async def sub_results_path_feed(app): logger.info('subscribed to results path feeds') camera_service: CameraService = app["camera_service"] feed = await camera_service.hub.subscribe("results_path") async for item in feed: logger.debug(item) await socket_io.emit("results_path", os.path.basename(item))
class TaskService: def __init__(self, root_path: Path): self.root_path = root_path self.running_tasks: MutableMapping[str, asyncio.Task] = {} async def _run_script(self, filename: str, queue: asyncio.Queue = None, /, **kwargs): script_path = f"{self.root_path}/{filename}" # process script arguments args = "" if "_" in kwargs: vs = kwargs.pop("_") args += " ".join([str(v) for v in vs]) args += " ".join([f"--{k}={v}" for k, v in kwargs.items()]) cmd = f"python -u {script_path} {args}" proc = await asyncio.create_subprocess_shell( cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) logger.info(f"Executing --- ({proc.pid}) `$ {cmd}`") try: while not proc.stdout.at_eof(): data = await proc.stdout.readline() line = data.decode("utf8").strip() if len(line) == 0: continue logger.info(f"({filename}) >>> {line}") if queue: await queue.put(line) exit_code = await proc.wait() logger.info(f'Finished ---- ({proc.pid}) `$ {cmd}` {exit_code=}') # raise exception if exit_code is not 0 if exit_code != 0: msg = 'Execution abort! Reason: ' # all stderr msg is buffered while not proc.stderr.at_eof(): data = await proc.stderr.readline() line = data.decode("utf8").strip() logger.error(f"({filename}) >>> {line}") msg += line raise Exception(msg) return exit_code except asyncio.CancelledError as e: logger.warning(f"Cancel process {cmd}({proc.pid})") proc.terminate() raise e
async def pub_squeue_items(app): camera_service: CameraService = app["camera_service"] try: await camera_service.emit_status_queue_item() except asyncio.CancelledError as e: logger.info('stopped pub squeue items') raise e
def _get_rpc_conn(self, rpc_config): rpc_host = "localhost" rpc_port = 51235 #rpc_config.get("host"), rpc_config.getint("port") conn = rpyc.connect(rpc_host, rpc_port) logger.info(f'Camera RPC connected! ({rpc_host}:{rpc_port})') return conn
async def sub_sensor_reading_feed(app): """ This should be called for only once """ logger.info('subscribed to sensor readings feeds') sensor_service: SensorService = app["sensor_service"] async for reading in sensor_service.on_reading(): await socket_io.emit("on_sensor_reading", reading.to_json())
async def stop_capturing(self, stop_script_name): payload = {"PSTOP": 1} await self.put_item(self.cmd_queue, payload) logger.info("Requested cqueue to stop capturing") exit_code = await self.task_service.run_script(stop_script_name) logger.debug(f"Ran stopcap script: {exit_code=}") return exit_code
async def stop(self): logger.debug('stopping detector') if self.debug: logger.debug( 'in debug mode, sleep for 1s, not really stop the detector via RPC' ) await asyncio.sleep(1) else: self.rpc.stopDetector() logger.info(f"stopped detector")
async def initiate_capturing(self, settings): csettings = settings.get("imaging") payload = { "PSTART": [ csettings.get("record.raw"), csettings.get("record.particle"), ] } await self.put_item(self.cmd_queue, payload) logger.info("Requested cqueue to start capturing")
async def sub_intensity_feed(app): logger.info('subscribed to intensity feeds') camera_service: CameraService = app["camera_service"] await camera_service.init_subscribers() try: async for item in camera_service.intensity_stream: await socket_io.emit("on_intensity_updated", item) except asyncio.CancelledError as e: logger.info('unsubscribe to intensity feed') raise e
def __init__(self, config): self.debug = config.getboolean("global", "DEBUG", fallback=True) self.config = config["detector_service"] self.thresholds = [ float(v) for v in self.config.get("THRESHOLDS").split(",") ] logger.info( f"thresholds {self.thresholds}") self.conn = self._get_conn() self.rpc = lazy(lambda: self.conn.root) self.hub: PubSub = AMemoryPubSub(asyncio.Queue)
def _get_queue_mgr(self, queue_config): queue_host = "localhost" queue_port = 51234 authkey ="wotwot".encode("utf-8") status_queue_name = "status_queue" cmd_queue_name = "cmd_queue" CameraQueueManager.register(status_queue_name) CameraQueueManager.register(cmd_queue_name) mgr = CameraQueueManager((queue_host, queue_port), authkey=authkey) # expensive! mgr.connect() logger.info(f"Queues connected! ({queue_host}:{queue_port})") return mgr
def _get_conn(self): # Setup the connection host = "localhost" port = 51237 conn = rpyc.connect( host, port, config={ "allow_pickle": True, "sync_request_timeout": self.config.getint("REQUEST_TIMEOUT", fallback=5), }, ) logger.info(f'Detector is connected! ({host}:{port})') return conn
async def emit_status_queue_item(self): # Start receiving item from RPC calls try: while True: item = await self.get_item(self.status_queue) if item == '--EOF--': break # logger.debug(f"Get squeue item: keys={item.keys()}") # Distribute item according to its topic if "CIMG" in item or "TIMG" in item: await self.hub.publish("image", item) elif "INT" in item: await self.hub.publish( "intensity", { "samples": item["INT"][0], "stats": { "fps": item["ISTAT"][0], "lptc": item["ISTAT"][1], }, }, ) elif "SPATH" in item: logger.info(f'get results path: {item["SPATH"]}') await self.hub.publish("results_path", item["SPATH"]) else: logger.info(f"get unhandled item: {item}") wait_for = float(self.config.get("QUEUE_CONSUME_RATE")) await asyncio.sleep(wait_for) except asyncio.CancelledError: logger.info('stopped pub squeue items') raise
def __init__( self, task_service: TaskService, setting_service: SettingService, detector_service: DetectorService, config: ConfigParser, ): # Hub for PubSub self.hub = AMemoryPubSub(asyncio.Queue) self.task_service = task_service self.setting_service = setting_service self.detector_service = detector_service self.rpc_conn = self._get_rpc_conn(config['camera_rpc']) self.rpc = lazy(lambda: self.rpc_conn.root) self.queue_mgr = self._get_queue_mgr(config["camera_queue"]) self.status_queue = lazy(lambda: self.queue_mgr.status_queue()) self.cmd_queue = lazy(lambda: self.queue_mgr.cmd_queue()) logger.info( f'qmgr should be lazily evaludated, not connected at this moment') self.config = config['camera_rpc']
async def start(self, path, monitor_mode): """ Start classifier. If monitor_mode is True, this function never ends and progress bar will stick to 49% It will emit event logs via centralized hub, event types are Progress and State. It will raise Exception if there is an issue while running, after State chagned to TaskState.Failed @TODO: add tests """ logger.info(f"start CG detection: {path=} {monitor_mode=}") p = Path(path) # path must be a directory if not p.exists(): raise Exception(f'Path not exists: {p}') if not p.is_dir(): raise Exception(f'Path is not directory: {p}') try: await self.hub.publish(EventTopics.State, self._event(TaskState.Ongoing)) # these calls can be blocking, consider run_in_executor self.rpc.stopDetector() self.rpc.startDetector(path, monitor_mode) for i in range(1, 5): child_dir = p / str(i) child_dir.mkdir(exist_ok=True) logger.info(f"created result directory: {child_dir}") counter = Counter() progress_value = 0 results = [] while progress_value < 100: idx, total_count = self.rpc.getPos() # handle empty or error cases if total_count == -1: logger.info('waiting for gathering samples') continue elif total_count == 0: logger.error(f"sample directory is empty: {p.resolve()}") break logger.debug(f'getting detection results') result = copy.deepcopy(self.rpc.getResults()) processed_count = len(result) logger.info(f"total processed counts: {processed_count}") progress_value = (idx + 1) / total_count * 100.0 results = results + result logger.info( f"detection progress: {progress_value}% ({idx}/{total_count})" ) await self.hub.publish( EventTopics.Logs, self._event( EventLogType.Progress, { 'progress': progress_value, 'processed': processed_count, 'total': total_count })) # wait for another round await asyncio.sleep(0.5) # 4 kinds of label triggered_samples = {k: [] for k in self.label_txt_mappings} #print(triggered_samples) for item in results: filename, label, confidence_level = item logger.info( f"{filename} : {label} {confidence_level}") label = int(label) label_txt = None try: label_txt = self.label_txt_mappings[label] except IndexError: raise IndexError( f'{label=} is invalid. Max label index is {len(self.label_txt_mappings)}' ) if label == 0: # skip item with label = 0 continue logger.info( f"found result: {filename=} {label=} {confidence_level=}") bname = Path(filename).stem if confidence_level >= self.thresholds[label]: counter[f'{label}|{label_txt}'] += 1 triggered_samples[label_txt].append({ 'idx': int(bname), 'confidence': confidence_level, 'label': label_txt }) logger.info( f"{counter=}, details={triggered_samples}" ) pathd = (p / str(label) / f"{ confidence_level }_{bname}.png") paths = p / filename shutil.copyfile(paths, pathd) logger.debug(f"copy {paths=} to {pathd=}") logger.info( f"completed CG detection: {counter=}, details={triggered_samples}" ) await self.hub.publish( EventTopics.Logs, self._event( EventLogType.Results, { 'results': triggered_samples, 'labelMapping': self.label_txt_mappings })) except Exception as e: logger.error(f"failed to run detector: {e}") await self.hub.publish(EventTopics.State, self._event(TaskState.Failed)) # propogate error to caller raise e finally: logger.debug(f'wait 2s before stopping detector') await asyncio.sleep(2) await self.stop() await self.hub.publish(EventTopics.State, self._event(TaskState.Completed))
async def put_item(self, queue: Queue, item): logger.info(f"send command: {item=}") return queue.put_nowait(item)
async def main(): done, pending = await asyncio.wait([foo(), bar()], return_when=asyncio.FIRST_COMPLETED) logger.info(f'{done}, {pending}')
async def update_camera_gain(self, gain: float): self.rpc.setGain(gain) logger.info(f"Updated camera {gain=}")
async def update_camera_exp(self, exposure: float): self.rpc.setExp(exposure) logger.info(f"Updated camera {exposure=}")
async def update_intensity_levels(self, low: int, high: int): logger.info(f"Updated intensity levels: {low=} {high=}") return await self.put_item(self.cmd_queue, {"PICH": [low, high]})
async def bar(): await asyncio.sleep(3) logger.info(f'bar: done')
async def start_auto_capturing(self, queue: asyncio.Queue, mode: str): """ Run the auto mode @todo: move to auto service """ settings = await self.setting_service.get() # Start capturing particle await self.initiate_capturing(settings) # Submit the task, this doesn't block tid = await self.initiate_capturing_script("startautoflow", f"{mode} {settings}", queue) detector_service_connected = self.detector_service.connected() classify_task = None try: if detector_service_connected: logger.info("detector is connected, try classifying images") # Detector is working, wait for the path to return # and start detecting sub = await self.hub.subscribe("results_path") monitor_mode = True try: # Get the first result path logger.info(f'waiting for the result path') path = await asyncio.wait_for(sub.__anext__(), timeout=3) logger.info( f"get results_path {path}, starting classifier") # Start detector classify_coro = self.detector_service.start( path, monitor_mode) classify_task = asyncio.create_task(classify_coro) except asyncio.TimeoutError: logger.error('get results_path timeout(3s)') detector_service_connected = False # clean up # wait for the tasks to be done script_result = await self.task_service.running_tasks[tid] logger.info(f"completed script: {script_result=}") await asyncio.sleep(5) except Exception as e: logger.exception(f"failed to clean up auto mode: {e}") finally: # shutdown camera await self.stop_capturing("stopautoflow") # shutdown detector if detector_service_connected and classify_task is not None: await asyncio.sleep(3) classify_task.cancel() try: await classify_task except asyncio.CancelledError: logger.info('Classification task is cancelled') logger.info('completed start autoflow task')
async def foo(): cnt = 0 while True: logger.info(f'foo: {cnt}') cnt += 1 await asyncio.sleep(1)
async def on_cleanup(app): logger.info('clean up app') scheduler = get_scheduler_from_app(app) logger.info(f'closing scheduler: {scheduler}') await scheduler.close()