class BasicActor(ABC): name: Optional[str] in_queue: Queue out_queue: Queue alive: Value _loop_task: Optional[Task] def __init__(self, name=None): self.name = name self._state = None ctx = SpawnContext() self.alive = Value('b', True) self.in_queue = Queue(ctx=ctx, maxsize=120) self.out_queue = Queue(ctx=ctx, maxsize=110) async def runner(self): loop = get_running_loop() self._state = await self.handle_started() self._loop_task = loop.create_task(self._do_loop(loop)) async def _do_loop(self, loop: AbstractEventLoop): while loop.is_running(): try: sent_from, message = self.in_queue.get(timeout=0.1) loop.create_task( self._handle_message(message, sent_from, self._state)) except Empty: pass await asyncio.sleep(0) def send_message(self, to, message): if self.out_queue.qsize() > 100: logger.warning("Shedding excess outgoing message") return self.out_queue.put((to, message)) async def _handle_message(self, message, sent_from, state): try: self._state = await self.handle_message(message, sent_from, state) except: self.stop() raise def stop(self): self.alive.value = False if self._loop_task: self._loop_task.cancel() @abstractmethod async def handle_message(self, message, sent_from, state) -> Any: pass @abstractmethod async def handle_started(self) -> Any: pass
def _centrallogger_worker(logging_q: mpq.Queue): """Entry for a thread providing central logging from subprocesses via a shared queue""" threadname = threading.currentThread().getName() thread_logger = LOGGER.getChild(threadname) thread_logger.info('{} started'.format(threadname)) while True: # only calling process terminates - keeping logging alive record = logging_q.get(block=True) if record == 'TER': thread_logger.info('received {}'.format(record)) break LOGGER.handle(record) qsize = logging_q.qsize() if qsize > 10: thread_logger.warning(f"logging_q has size {qsize}") if logging_q.full(): thread_logger.warning(f"logging_q is full")
def yk_monitor(self, mon_l): # forming command to run parallel monitoring processes mon_cmd = ' & '.join(["xinput test {}".format(y_id) for y_id in mon_l]) monitor = subprocess.Popen(mon_cmd, shell=True, stdout=subprocess.PIPE) stdout_queue = Queue() stdout_reader = AsynchronousFileReader(monitor.stdout, stdout_queue) stdout_reader.start() triggered = False timestamp = time.time() while not stdout_reader.eof and time.time() - timestamp < TIMEOUT: while stdout_queue.qsize() > 0: stdout_queue.get() # emptying queue triggered = True time.sleep(.01) if triggered: print('YubiKey triggered. Now disabling.') break time.sleep(.001) if not triggered: print('No YubiKey triggered. Timeout.')
def yk_monitor(self, mon_l): # forming command to run parallel monitoring processes mon_cmd = ' & '.join(["xinput test {}".format(y_id) for y_id in mon_l]) monitor = subprocess.Popen(mon_cmd, shell=True, stdout=subprocess.PIPE) stdout_queue = Queue() stdout_reader = AsynchronousFileReader(monitor.stdout, stdout_queue) stdout_reader.start() triggered = False timestamp = time.time() while not stdout_reader.eof and time.time() - timestamp < TIMEOUT: while stdout_queue.qsize() > 0: stdout_queue.get() # emptying queue triggered = True time.sleep(.04) if triggered: print('YubiKey triggered. Now disabling.') break time.sleep(.001) if not triggered: print('No YubiKey triggered. Timeout.')
class YubiGuard: def __init__(self, scrlck_mode=False): self.scrlck_mode = scrlck_mode self.id_q = Queue() self.on_q = Queue() self.pi_q = Queue() # init processes gi_proc = Process(target=self.get_ids) gi_proc.daemon = True cs_proc = Process(target=self.change_state) # no daemon, or main program will terminate before Keys can be unlocked cs_proc.daemon = False zmq_lis = ZmqListener( self.on_q) # somehow works ony with threads not processes zmq_lis_thr = Thread(target=zmq_lis.start_listener) zmq_lis_thr.setDaemon(True) pi = PanelIndicator(self.pi_q, self.on_q) # starting processes and catching exceptions: try: gi_proc.start() cs_proc.start() zmq_lis_thr.start() pi.run_pi() # main loop of root process except (KeyboardInterrupt, SystemExit): print('Caught exit event.') finally: # send exit signal, will reactivate YubiKey slots print('Sending EXIT_SIGNAL') self.on_q.put(EXIT_SIGNAL) def get_ids(self): old_id_l = [] no_key = True pat = re.compile(r"(?:Yubikey.*?id=)(\d+)", re.IGNORECASE) while True: new_id_l = [] # get list of xinput device ids and extract those of YubiKeys: xinput = shell_this('xinput list') matches = re.findall(pat, xinput) new_id_l.extend(matches) new_id_l.sort() if not new_id_l and not no_key: self.pi_q.put(NOKEY_SIGNAL) print('No YubiKey(s) detected.') no_key = True elif new_id_l and no_key: self.pi_q.put(OFF_SIGNAL) print('YubiKey(s) detected.') no_key = False # notify: msg_cmd = """notify-send --expire-time=2000 \ 'YubiKey(s) detected.'""" shell_this(msg_cmd) if new_id_l != old_id_l: print('Change in YubiKey ids detected. From {} to {}.'.format( old_id_l, new_id_l)) self.id_q.put(new_id_l) # lock screen if screenlock and YubiKey is removed: if self.scrlck_mode and len(new_id_l) < len(old_id_l): print('Locking screen.') shell_this(get_scrlck_cmd()) # execute screen lock command old_id_l = new_id_l time.sleep(.1) def turn_keys(self, id_l, lock=True ): # problem of value loss of cs_id_l found in this function tk_id_l = id_l if lock: print('Locking YubiKey(s).') state_flag = '0' self.pi_q.put(OFF_SIGNAL) else: print('Unlocking YubiKey(s).') state_flag = '1' self.pi_q.put(ON_SIGNAL) shell_this('; '.join(["xinput set-int-prop {} \"Device Enabled\" 8 {}". format(tk_id, state_flag) for tk_id in tk_id_l])) def check_state(self, check_id_l): # check if all states have indeed changed: pat = re.compile(r"(?:Device Enabled.+?:).?([01])", re.IGNORECASE) # check if state has indeed changed: for tk_id in check_id_l: sh_out = shell_this('xinput list-props {}'.format(tk_id)) match = re.search(pat, sh_out) if match: if match.group(1) != '0': return False def change_state(self): cs_id_l = [] cs_signal = '' while True: # retrieve input from queues while self.id_q.qsize() > 0: cs_id_l = self.id_q.get() while self.on_q.qsize() > 0: cs_signal = self.on_q.get() # not accepting any more signals if cs_signal == EXIT_SIGNAL: self.turn_keys(cs_id_l, lock=False) sys.exit(0) # lock/unlock if cs_id_l: if cs_signal == ON_SIGNAL: self.turn_keys(cs_id_l, lock=False) mon_thread = Thread( target=self.yk_monitor, args=(cs_id_l, )) mon_thread.start() mon_thread.join() # putting in separator, nullifying all preceding ON_SIGNALS # to prevent possible over-triggering: self.on_q.put('') elif self.check_state( cs_id_l) is False: # lock keys if they are unlocked self.turn_keys(cs_id_l, lock=True) # reset state to prevent continued unlocking/locking cs_signal = '' time.sleep(.01) def yk_monitor(self, mon_l): # forming command to run parallel monitoring processes mon_cmd = ' & '.join(["xinput test {}".format(y_id) for y_id in mon_l]) monitor = subprocess.Popen(mon_cmd, shell=True, stdout=subprocess.PIPE) stdout_queue = Queue() stdout_reader = AsynchronousFileReader(monitor.stdout, stdout_queue) stdout_reader.start() triggered = False timestamp = time.time() while not stdout_reader.eof and time.time() - timestamp < TIMEOUT: while stdout_queue.qsize() > 0: stdout_queue.get() # emptying queue triggered = True time.sleep(.01) if triggered: print('YubiKey triggered. Now disabling.') break time.sleep(.001) if not triggered: print('No YubiKey triggered. Timeout.')
from xing_tick_crawler.crawler import stock_market_crawler, futures_option_market_crawler from datetime import datetime from multiprocessing import Process, get_context from multiprocessing.queues import Queue if __name__ == "__main__": stock_market_subs_option = { # 주식 VI 정보 off 'STOCK_VI_ON_OFF': False, } futures_option_market_subs_option = {} queue = Queue(ctx=get_context()) p0 = Process(target=stock_market_crawler, args=(queue, ), kwargs=stock_market_subs_option) p1 = Process(target=futures_option_market_crawler, args=(queue, ), kwargs=futures_option_market_subs_option) p0.start() p1.start() while True: tick = queue.get() waiting_tasks = queue.qsize() tick_type, tick_data = tick print(f"\r{datetime.now()} waiting tasks : {'%6d' % waiting_tasks}", end='') print(tick_type, tick_data)
class YubiGuard: def __init__(self, scrlck_mode=False): self.scrlck_mode = scrlck_mode self.id_q = Queue() self.on_q = Queue() self.pi_q = Queue() # init processes gi_proc = Process(target=self.get_ids) gi_proc.daemon = True cs_proc = Process(target=self.change_state) # no daemon, or main program will terminate before Keys can be unlocked cs_proc.daemon = False zmq_lis = ZmqListener( self.on_q) # somehow works ony with threads not processes zmq_lis_thr = Thread(target=zmq_lis.start_listener) zmq_lis_thr.setDaemon(True) pi = PanelIndicator(self.pi_q, self.on_q) # starting processes and catching exceptions: try: gi_proc.start() cs_proc.start() zmq_lis_thr.start() pi.run_pi() # main loop of root process except (KeyboardInterrupt, SystemExit): print('Caught exit event.') finally: # send exit signal, will reactivate YubiKey slots print('Sending EXIT_SIGNAL') self.on_q.put(EXIT_SIGNAL) def get_ids(self): old_id_l = [] no_key = True pat = re.compile(r"(?:Yubikey.*?id=)(\d+)", re.IGNORECASE) while True: new_id_l = [] # get list of xinput device ids and extract those of YubiKeys: xinput = shell_this('xinput list') matches = re.findall(pat, xinput) new_id_l.extend(matches) new_id_l.sort() if not new_id_l and not no_key: self.pi_q.put(NOKEY_SIGNAL) print('No YubiKey(s) detected.') no_key = True elif new_id_l and no_key: self.pi_q.put(OFF_SIGNAL) print('YubiKey(s) detected.') no_key = False # notify: msg_cmd = """notify-send --expire-time=2000 \ 'YubiKey(s) detected.'""" shell_this(msg_cmd) if new_id_l != old_id_l: print('Change in YubiKey ids detected. From {} to {}.'.format( old_id_l, new_id_l)) self.id_q.put(new_id_l) # lock screen if screenlock and YubiKey is removed: if self.scrlck_mode and len(new_id_l) < len(old_id_l): print('Locking screen.') shell_this(get_scrlck_cmd()) # execute screen lock command old_id_l = new_id_l time.sleep(.1) def turn_keys(self, id_l, lock=True ): # problem of value loss of cs_id_l found in this function tk_id_l = id_l if lock: print('Locking YubiKey(s).') state_flag = '0' self.pi_q.put(OFF_SIGNAL) else: print('Unlocking YubiKey(s).') state_flag = '1' self.pi_q.put(ON_SIGNAL) shell_this('; '.join(["xinput set-int-prop {} \"Device Enabled\" 8 {}". format(tk_id, state_flag) for tk_id in tk_id_l])) def check_state(self, check_id_l): # check if all states have indeed changed: pat = re.compile(r"(?:Device Enabled.+?:).?([01])", re.IGNORECASE) # check if state has indeed changed: for tk_id in check_id_l: sh_out = shell_this('xinput list-props {}'.format(tk_id)) match = re.search(pat, sh_out) if match: if match.group(1) != '0': return False def change_state(self): cs_id_l = [] cs_signal = '' while True: # retrieve input from queues while self.id_q.qsize() > 0: cs_id_l = self.id_q.get() while self.on_q.qsize() > 0: cs_signal = self.on_q.get() # not accepting any more signals if cs_signal == EXIT_SIGNAL: self.turn_keys(cs_id_l, lock=False) sys.exit(0) # lock/unlock if cs_id_l: if cs_signal == ON_SIGNAL: self.turn_keys(cs_id_l, lock=False) mon_thread = Thread( target=self.yk_monitor, args=(cs_id_l, )) mon_thread.start() mon_thread.join() # putting in separator, nullifying all preceding ON_SIGNALS # to prevent possible over-triggering: self.on_q.put('') elif self.check_state( cs_id_l) is False: # lock keys if they are unlocked self.turn_keys(cs_id_l, lock=True) # reset state to prevent continued unlocking/locking cs_signal = '' time.sleep(.01) def yk_monitor(self, mon_l): # forming command to run parallel monitoring processes mon_cmd = ' & '.join(["xinput test {}".format(y_id) for y_id in mon_l]) monitor = subprocess.Popen(mon_cmd, shell=True, stdout=subprocess.PIPE) stdout_queue = Queue() stdout_reader = AsynchronousFileReader(monitor.stdout, stdout_queue) stdout_reader.start() triggered = False timestamp = time.time() while not stdout_reader.eof and time.time() - timestamp < TIMEOUT: while stdout_queue.qsize() > 0: stdout_queue.get() # emptying queue triggered = True time.sleep(.04) if triggered: print('YubiKey triggered. Now disabling.') break time.sleep(.001) if not triggered: print('No YubiKey triggered. Timeout.')