async def start_auto_capturing(self, queue: asyncio.Queue, mode: str):
"""
Run the auto mode
@todo: move to auto service
"""
settings = await self.setting_service.get()
# Start capturing particle
await self.initiate_capturing(settings)
# Submit the task, this doesn't block
tid = await self.initiate_capturing_script("startautoflow", f"{mode} {settings}",
queue)
detector_service_connected = self.detector_service.connected()
classify_task = None
try:
if detector_service_connected:
logger.info("detector is connected, try classifying images")
# Detector is working, wait for the path to return
# and start detecting
sub = await self.hub.subscribe("results_path")
monitor_mode = True
try:
# Get the first result path
logger.info(f'waiting for the result path')
path = await asyncio.wait_for(sub.__anext__(), timeout=3)
logger.info(
f"get results_path {path}, starting classifier")
# Start detector
classify_coro = self.detector_service.start(
path, monitor_mode)
classify_task = asyncio.create_task(classify_coro)
except asyncio.TimeoutError:
logger.error('get results_path timeout(3s)')
detector_service_connected = False
# clean up
# wait for the tasks to be done
script_result = await self.task_service.running_tasks[tid]
logger.info(f"completed script: {script_result=}")
await asyncio.sleep(5)
except Exception as e:
logger.exception(f"failed to clean up auto mode: {e}")
finally:
# shutdown camera
await self.stop_capturing("stopautoflow")
# shutdown detector
if detector_service_connected and classify_task is not None:
await asyncio.sleep(3)
classify_task.cancel()
try:
await classify_task
except asyncio.CancelledError:
logger.info('Classification task is cancelled')
logger.info('completed start autoflow task')
def connected(self):
try:
self.conn.ping()
return True
except Exception:
host, port = self.config.get('HOST'), self.config.getint('PORT')
logger.error(f'Detector RPC is down! ({host}:{port})')
return False
class TaskService:
def __init__(self, root_path: Path):
self.root_path = root_path
self.running_tasks: MutableMapping[str, asyncio.Task] = {}
async def _run_script(self,
filename: str,
queue: asyncio.Queue = None,
/,
**kwargs):
script_path = f"{self.root_path}/{filename}"
# process script arguments
args = ""
if "_" in kwargs:
vs = kwargs.pop("_")
args += " ".join([str(v) for v in vs])
args += " ".join([f"--{k}={v}" for k, v in kwargs.items()])
cmd = f"python -u {script_path} {args}"
proc = await asyncio.create_subprocess_shell(
cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE)
logger.info(f"Executing --- ({proc.pid}) `$ {cmd}`")
try:
while not proc.stdout.at_eof():
data = await proc.stdout.readline()
line = data.decode("utf8").strip()
if len(line) == 0:
continue
logger.info(f"({filename}) >>> {line}")
if queue:
await queue.put(line)
exit_code = await proc.wait()
logger.info(f'Finished ---- ({proc.pid}) `$ {cmd}` {exit_code=}')
# raise exception if exit_code is not 0
if exit_code != 0:
msg = 'Execution abort! Reason: '
# all stderr msg is buffered
while not proc.stderr.at_eof():
data = await proc.stderr.readline()
line = data.decode("utf8").strip()
logger.error(f"({filename}) >>> {line}")
msg += line
raise Exception(msg)
return exit_code
except asyncio.CancelledError as e:
logger.warning(f"Cancel process {cmd}({proc.pid})")
proc.terminate()
raise e
def connected(self) -> bool:
# Check RPC status
try:
self.rpc_conn.ping()
except Exception as e:
logger.error(f'Camera RPC is down! {e}')
return False
# Check queue status
try:
self.queue_mgr.connect()
except Exception as e:
logger.error(f'Camera QUEUE is down! ({e})')
return False
return True
async def start_manual_capturing(self) -> str:
try:
settings = await self.setting_service.get()
# Step 1: send item to cqueue requesting start capturing
await self.initiate_capturing(settings)
tid = await self.initiate_capturing_script("mfs_pd", settings)
return await self.task_service.get(tid)
except CancelledError as e:
logger.warning('cancelling manual capturing')
raise e
except Exception as e:
logger.error(f'failed to run manual capturing: {e}')
raise e
finally:
await self.stop_capturing('mfs_stop')
async def start(self, path, monitor_mode):
"""
Start classifier. If monitor_mode is True, this function never ends and
progress bar will stick to 49%
It will emit event logs via centralized hub, event types are Progress and State.
It will raise Exception if there is an issue while running, after State chagned to TaskState.Failed
@TODO: add tests
"""
logger.info(f"start CG detection: {path=} {monitor_mode=}")
p = Path(path)
# path must be a directory
if not p.exists():
raise Exception(f'Path not exists: {p}')
if not p.is_dir():
raise Exception(f'Path is not directory: {p}')
try:
await self.hub.publish(EventTopics.State,
self._event(TaskState.Ongoing))
# these calls can be blocking, consider run_in_executor
self.rpc.stopDetector()
self.rpc.startDetector(path, monitor_mode)
for i in range(1, 5):
child_dir = p / str(i)
child_dir.mkdir(exist_ok=True)
logger.info(f"created result directory: {child_dir}")
counter = Counter()
progress_value = 0
results = []
while progress_value < 100:
idx, total_count = self.rpc.getPos()
# handle empty or error cases
if total_count == -1:
logger.info('waiting for gathering samples')
continue
elif total_count == 0:
logger.error(f"sample directory is empty: {p.resolve()}")
break
logger.debug(f'getting detection results')
result = copy.deepcopy(self.rpc.getResults())
processed_count = len(result)
logger.info(f"total processed counts: {processed_count}")
progress_value = (idx + 1) / total_count * 100.0
results = results + result
logger.info(
f"detection progress: {progress_value}% ({idx}/{total_count})"
)
await self.hub.publish(
EventTopics.Logs,
self._event(
EventLogType.Progress, {
'progress': progress_value,
'processed': processed_count,
'total': total_count
}))
# wait for another round
await asyncio.sleep(0.5)
# 4 kinds of label
triggered_samples = {k: [] for k in self.label_txt_mappings}
#print(triggered_samples)
for item in results:
filename, label, confidence_level = item
logger.info(
f"{filename} : {label} {confidence_level}")
label = int(label)
label_txt = None
try:
label_txt = self.label_txt_mappings[label]
except IndexError:
raise IndexError(
f'{label=} is invalid. Max label index is {len(self.label_txt_mappings)}'
)
if label == 0:
# skip item with label = 0
continue
logger.info(
f"found result: {filename=} {label=} {confidence_level=}")
bname = Path(filename).stem
if confidence_level >= self.thresholds[label]:
counter[f'{label}|{label_txt}'] += 1
triggered_samples[label_txt].append({
'idx': int(bname),
'confidence': confidence_level,
'label': label_txt
})
logger.info(
f"{counter=}, details={triggered_samples}"
)
pathd = (p / str(label) / f"{ confidence_level }_{bname}.png")
paths = p / filename
shutil.copyfile(paths, pathd)
logger.debug(f"copy {paths=} to {pathd=}")
logger.info(
f"completed CG detection: {counter=}, details={triggered_samples}"
)
await self.hub.publish(
EventTopics.Logs,
self._event(
EventLogType.Results, {
'results': triggered_samples,
'labelMapping': self.label_txt_mappings
}))
except Exception as e:
logger.error(f"failed to run detector: {e}")
await self.hub.publish(EventTopics.State,
self._event(TaskState.Failed))
# propogate error to caller
raise e
finally:
logger.debug(f'wait 2s before stopping detector')
await asyncio.sleep(2)
await self.stop()
await self.hub.publish(EventTopics.State,
self._event(TaskState.Completed))