async def sub_results_path_feed(app):
logger.info('subscribed to results path feeds')
camera_service: CameraService = app["camera_service"]
feed = await camera_service.hub.subscribe("results_path")
async for item in feed:
logger.debug(item)
await socket_io.emit("results_path", os.path.basename(item))
class TaskService:
def __init__(self, root_path: Path):
self.root_path = root_path
self.running_tasks: MutableMapping[str, asyncio.Task] = {}
async def _run_script(self,
filename: str,
queue: asyncio.Queue = None,
/,
**kwargs):
script_path = f"{self.root_path}/{filename}"
# process script arguments
args = ""
if "_" in kwargs:
vs = kwargs.pop("_")
args += " ".join([str(v) for v in vs])
args += " ".join([f"--{k}={v}" for k, v in kwargs.items()])
cmd = f"python -u {script_path} {args}"
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
logger.info(f"Executing --- ({proc.pid}) `$ {cmd}`")
try:
while not proc.stdout.at_eof():
data = await proc.stdout.readline()
line = data.decode("utf8").strip()
if len(line) == 0:
continue
logger.info(f"({filename}) >>> {line}")
if queue:
await queue.put(line)
exit_code = await proc.wait()
logger.info(f'Finished ---- ({proc.pid}) `$ {cmd}` {exit_code=}')
# raise exception if exit_code is not 0
if exit_code != 0:
msg = 'Execution abort! Reason: '
# all stderr msg is buffered
while not proc.stderr.at_eof():
data = await proc.stderr.readline()
line = data.decode("utf8").strip()
logger.error(f"({filename}) >>> {line}")
msg += line
raise Exception(msg)
return exit_code
except asyncio.CancelledError as e:
logger.warning(f"Cancel process {cmd}({proc.pid})")
proc.terminate()
raise e
async def pub_squeue_items(app):
camera_service: CameraService = app["camera_service"]
try:
await camera_service.emit_status_queue_item()
except asyncio.CancelledError as e:
logger.info('stopped pub squeue items')
raise e
def _get_rpc_conn(self, rpc_config):
rpc_host = "localhost"
rpc_port = 51235
#rpc_config.get("host"), rpc_config.getint("port")
conn = rpyc.connect(rpc_host, rpc_port)
logger.info(f'Camera RPC connected! ({rpc_host}:{rpc_port})')
return conn
async def sub_sensor_reading_feed(app):
"""
This should be called for only once
"""
logger.info('subscribed to sensor readings feeds')
sensor_service: SensorService = app["sensor_service"]
async for reading in sensor_service.on_reading():
await socket_io.emit("on_sensor_reading", reading.to_json())
async def stop_capturing(self, stop_script_name):
payload = {"PSTOP": 1}
await self.put_item(self.cmd_queue, payload)
logger.info("Requested cqueue to stop capturing")
exit_code = await self.task_service.run_script(stop_script_name)
logger.debug(f"Ran stopcap script: {exit_code=}")
return exit_code
async def stop(self):
logger.debug('stopping detector')
if self.debug:
logger.debug(
'in debug mode, sleep for 1s, not really stop the detector via RPC'
)
await asyncio.sleep(1)
else:
self.rpc.stopDetector()
logger.info(f"stopped detector")
async def initiate_capturing(self, settings):
csettings = settings.get("imaging")
payload = {
"PSTART": [
csettings.get("record.raw"),
csettings.get("record.particle"),
]
}
await self.put_item(self.cmd_queue, payload)
logger.info("Requested cqueue to start capturing")
async def sub_intensity_feed(app):
logger.info('subscribed to intensity feeds')
camera_service: CameraService = app["camera_service"]
await camera_service.init_subscribers()
try:
async for item in camera_service.intensity_stream:
await socket_io.emit("on_intensity_updated", item)
except asyncio.CancelledError as e:
logger.info('unsubscribe to intensity feed')
raise e
def __init__(self, config):
self.debug = config.getboolean("global", "DEBUG", fallback=True)
self.config = config["detector_service"]
self.thresholds = [
float(v) for v in self.config.get("THRESHOLDS").split(",")
]
logger.info(
f"thresholds {self.thresholds}")
self.conn = self._get_conn()
self.rpc = lazy(lambda: self.conn.root)
self.hub: PubSub = AMemoryPubSub(asyncio.Queue)
def _get_queue_mgr(self, queue_config):
queue_host = "localhost"
queue_port = 51234
authkey ="wotwot".encode("utf-8")
status_queue_name = "status_queue"
cmd_queue_name = "cmd_queue"
CameraQueueManager.register(status_queue_name)
CameraQueueManager.register(cmd_queue_name)
mgr = CameraQueueManager((queue_host, queue_port), authkey=authkey)
# expensive!
mgr.connect()
logger.info(f"Queues connected! ({queue_host}:{queue_port})")
return mgr
def _get_conn(self):
# Setup the connection
host = "localhost"
port = 51237
conn = rpyc.connect(
host,
port,
config={
"allow_pickle":
True,
"sync_request_timeout":
self.config.getint("REQUEST_TIMEOUT", fallback=5),
},
)
logger.info(f'Detector is connected! ({host}:{port})')
return conn
async def emit_status_queue_item(self):
# Start receiving item from RPC calls
try:
while True:
item = await self.get_item(self.status_queue)
if item == '--EOF--':
break
# logger.debug(f"Get squeue item: keys={item.keys()}")
# Distribute item according to its topic
if "CIMG" in item or "TIMG" in item:
await self.hub.publish("image", item)
elif "INT" in item:
await self.hub.publish(
"intensity",
{
"samples": item["INT"][0],
"stats": {
"fps": item["ISTAT"][0],
"lptc": item["ISTAT"][1],
},
},
)
elif "SPATH" in item:
logger.info(f'get results path: {item["SPATH"]}')
await self.hub.publish("results_path", item["SPATH"])
else:
logger.info(f"get unhandled item: {item}")
wait_for = float(self.config.get("QUEUE_CONSUME_RATE"))
await asyncio.sleep(wait_for)
except asyncio.CancelledError:
logger.info('stopped pub squeue items')
raise
def __init__(
self,
task_service: TaskService,
setting_service: SettingService,
detector_service: DetectorService,
config: ConfigParser,
):
# Hub for PubSub
self.hub = AMemoryPubSub(asyncio.Queue)
self.task_service = task_service
self.setting_service = setting_service
self.detector_service = detector_service
self.rpc_conn = self._get_rpc_conn(config['camera_rpc'])
self.rpc = lazy(lambda: self.rpc_conn.root)
self.queue_mgr = self._get_queue_mgr(config["camera_queue"])
self.status_queue = lazy(lambda: self.queue_mgr.status_queue())
self.cmd_queue = lazy(lambda: self.queue_mgr.cmd_queue())
logger.info(
f'qmgr should be lazily evaludated, not connected at this moment')
self.config = config['camera_rpc']
async def start(self, path, monitor_mode):
"""
Start classifier. If monitor_mode is True, this function never ends and
progress bar will stick to 49%
It will emit event logs via centralized hub, event types are Progress and State.
It will raise Exception if there is an issue while running, after State chagned to TaskState.Failed
@TODO: add tests
"""
logger.info(f"start CG detection: {path=} {monitor_mode=}")
p = Path(path)
# path must be a directory
if not p.exists():
raise Exception(f'Path not exists: {p}')
if not p.is_dir():
raise Exception(f'Path is not directory: {p}')
try:
await self.hub.publish(EventTopics.State,
self._event(TaskState.Ongoing))
# these calls can be blocking, consider run_in_executor
self.rpc.stopDetector()
self.rpc.startDetector(path, monitor_mode)
for i in range(1, 5):
child_dir = p / str(i)
child_dir.mkdir(exist_ok=True)
logger.info(f"created result directory: {child_dir}")
counter = Counter()
progress_value = 0
results = []
while progress_value < 100:
idx, total_count = self.rpc.getPos()
# handle empty or error cases
if total_count == -1:
logger.info('waiting for gathering samples')
continue
elif total_count == 0:
logger.error(f"sample directory is empty: {p.resolve()}")
break
logger.debug(f'getting detection results')
result = copy.deepcopy(self.rpc.getResults())
processed_count = len(result)
logger.info(f"total processed counts: {processed_count}")
progress_value = (idx + 1) / total_count * 100.0
results = results + result
logger.info(
f"detection progress: {progress_value}% ({idx}/{total_count})"
)
await self.hub.publish(
EventTopics.Logs,
self._event(
EventLogType.Progress, {
'progress': progress_value,
'processed': processed_count,
'total': total_count
}))
# wait for another round
await asyncio.sleep(0.5)
# 4 kinds of label
triggered_samples = {k: [] for k in self.label_txt_mappings}
#print(triggered_samples)
for item in results:
filename, label, confidence_level = item
logger.info(
f"{filename} : {label} {confidence_level}")
label = int(label)
label_txt = None
try:
label_txt = self.label_txt_mappings[label]
except IndexError:
raise IndexError(
f'{label=} is invalid. Max label index is {len(self.label_txt_mappings)}'
)
if label == 0:
# skip item with label = 0
continue
logger.info(
f"found result: {filename=} {label=} {confidence_level=}")
bname = Path(filename).stem
if confidence_level >= self.thresholds[label]:
counter[f'{label}|{label_txt}'] += 1
triggered_samples[label_txt].append({
'idx': int(bname),
'confidence': confidence_level,
'label': label_txt
})
logger.info(
f"{counter=}, details={triggered_samples}"
)
pathd = (p / str(label) / f"{ confidence_level }_{bname}.png")
paths = p / filename
shutil.copyfile(paths, pathd)
logger.debug(f"copy {paths=} to {pathd=}")
logger.info(
f"completed CG detection: {counter=}, details={triggered_samples}"
)
await self.hub.publish(
EventTopics.Logs,
self._event(
EventLogType.Results, {
'results': triggered_samples,
'labelMapping': self.label_txt_mappings
}))
except Exception as e:
logger.error(f"failed to run detector: {e}")
await self.hub.publish(EventTopics.State,
self._event(TaskState.Failed))
# propogate error to caller
raise e
finally:
logger.debug(f'wait 2s before stopping detector')
await asyncio.sleep(2)
await self.stop()
await self.hub.publish(EventTopics.State,
self._event(TaskState.Completed))
async def put_item(self, queue: Queue, item):
logger.info(f"send command: {item=}")
return queue.put_nowait(item)
async def main():
done, pending = await asyncio.wait([foo(), bar()],
return_when=asyncio.FIRST_COMPLETED)
logger.info(f'{done}, {pending}')
async def update_camera_gain(self, gain: float):
self.rpc.setGain(gain)
logger.info(f"Updated camera {gain=}")
async def update_camera_exp(self, exposure: float):
self.rpc.setExp(exposure)
logger.info(f"Updated camera {exposure=}")
async def update_intensity_levels(self, low: int, high: int):
logger.info(f"Updated intensity levels: {low=} {high=}")
return await self.put_item(self.cmd_queue, {"PICH": [low, high]})
async def bar():
await asyncio.sleep(3)
logger.info(f'bar: done')
async def start_auto_capturing(self, queue: asyncio.Queue, mode: str):
"""
Run the auto mode
@todo: move to auto service
"""
settings = await self.setting_service.get()
# Start capturing particle
await self.initiate_capturing(settings)
# Submit the task, this doesn't block
tid = await self.initiate_capturing_script("startautoflow", f"{mode} {settings}",
queue)
detector_service_connected = self.detector_service.connected()
classify_task = None
try:
if detector_service_connected:
logger.info("detector is connected, try classifying images")
# Detector is working, wait for the path to return
# and start detecting
sub = await self.hub.subscribe("results_path")
monitor_mode = True
try:
# Get the first result path
logger.info(f'waiting for the result path')
path = await asyncio.wait_for(sub.__anext__(), timeout=3)
logger.info(
f"get results_path {path}, starting classifier")
# Start detector
classify_coro = self.detector_service.start(
path, monitor_mode)
classify_task = asyncio.create_task(classify_coro)
except asyncio.TimeoutError:
logger.error('get results_path timeout(3s)')
detector_service_connected = False
# clean up
# wait for the tasks to be done
script_result = await self.task_service.running_tasks[tid]
logger.info(f"completed script: {script_result=}")
await asyncio.sleep(5)
except Exception as e:
logger.exception(f"failed to clean up auto mode: {e}")
finally:
# shutdown camera
await self.stop_capturing("stopautoflow")
# shutdown detector
if detector_service_connected and classify_task is not None:
await asyncio.sleep(3)
classify_task.cancel()
try:
await classify_task
except asyncio.CancelledError:
logger.info('Classification task is cancelled')
logger.info('completed start autoflow task')
async def foo():
cnt = 0
while True:
logger.info(f'foo: {cnt}')
cnt += 1
await asyncio.sleep(1)
async def on_cleanup(app):
logger.info('clean up app')
scheduler = get_scheduler_from_app(app)
logger.info(f'closing scheduler: {scheduler}')
await scheduler.close()