def __init__(self, config: ConfigContextCommon) -> None:
"""
Args:
config: A config object.
"""
logname = pathlib.Path(__file__).stem
self._logger = logging.getLogger(f"{config.PACKAGE_NAME}.{logname}")
self._logger.debug("")
self._tuuid = TUUIDCommon(config)
def __init__(self, ctx: ContextBase) -> None:
"""
Args:
ctx: The Context object.
"""
super().__init__(daemon=True) # terminate together w main thread
self._logger = ctx._logger
self._logger.info("Results")
self._util_cmd = ctx.util_cmd
self._resultq = ctx.resultq
self._json = JSONCommon(ctx)
self._tuuid = TUUIDCommon(ctx)
self._failfast_ev = ctx.failfast_ev
self._startfast_br = ctx.startfast_br
def __init__(self, ctx: ContextBase) -> None:
"""
Args:
ctx: The Context object.
"""
super().__init__(daemon=True) # terminate together w main thread
self._logger = ctx._logger
self._logger.info("Incoming")
self._workq = ctx.workq
self._util_cmd = ctx.util_cmd
self._timeout = ctx.timeout
self._json = JSONCommon(ctx)
self._tuuid = TUUIDCommon(ctx)
self._check_interval_secs = ctx.check_interval_secs
self._lastcheck_time = float(0) # init to zero
self._incoming_dirp = ctx.INCOMING_DIRP
self._valid_extensions = ctx.VALID_EXTENSIONS
self._failfast_ev = ctx.failfast_ev
self._startfast_br = ctx.startfast_br
def __init__(self, ctx: ContextBase, worker: int) -> None:
"""
Args:
ctx: The Context object.
worker: The worker index.
"""
super().__init__(daemon=True) # worker terminates if parent does
self._logger = ctx._logger
self._logger.info(f"worker: {worker}")
self._worker = worker
self._workq = ctx.workq
self._resultq = ctx.resultq
self._tuuid = TUUIDCommon(ctx)
self._transforms = TransformsDemo(ctx)
self._timeout = ctx.timeout
self._modified_dirp = ctx.MODIFIED_DIRP
self._original_dirp = ctx.ORIGINAL_DIRP
self._rejected_dirp = ctx.REJECTED_DIRP
self._failfast_ev = ctx.failfast_ev
self._startfast_br = ctx.startfast_br
class JSONCommon(object):
"""
Helper class for working with JSON.
"""
def __init__(self, config: ConfigContextCommon) -> None:
"""
Args:
config: A config object.
"""
logname = pathlib.Path(__file__).stem
self._logger = logging.getLogger(f"{config.PACKAGE_NAME}.{logname}")
self._logger.debug("")
self._tuuid = TUUIDCommon(config)
def jsonsafe(self, val: Any) -> Any:
"""
Create a safe version of the input value.
Args:
val: A value of any type.
Returns:
The value as a type suitable for JSON conversion or database
insert.
"""
newval: Any
if isinstance(val, bool):
newval = "true" if val else "false"
elif isinstance(val, (str, int, float)) or val is None:
newval = val
elif isinstance(val, datetime):
newval = self._tuuid.get_tza_utcdt_isoz(val)
elif isinstance(val, timedelta):
newval = val.total_seconds()
elif isinstance(val, bytes):
newval = val.decode("ascii") # e.g. for hash
elif isinstance(val, dict):
newval = {k: self.jsonsafe(v) for k, v in val.items()}
elif isinstance(val, list):
newval = [self.jsonsafe(v) for v in val]
elif isinstance(val, tuple):
newval = tuple(map(self.jsonsafe, val))
else: # if all else fails... (e.g. TimeUUID)
newval = str(val)
return newval
def jsondumps(self, val: Any) -> str:
"""
Invoke json.dumps on the 'jsonsafe'd input value w standard options.
Args:
val: A value of any type.
Returns:
A JSON string.
"""
return json.dumps(self.jsonsafe(val),
ensure_ascii=False,
indent=2,
sort_keys=True)
class ResultsDemo(Thread):
"""
Handle ResultNT messages from Workers.
NOTE: we use the Results thread to acknowledge Pub/Sub messages in addition
to the logging that you see below. We also use this thread to indicate to our
Redis cache that the work has completed.
"""
def __init__(self, ctx: ContextBase) -> None:
"""
Args:
ctx: The Context object.
"""
super().__init__(daemon=True) # terminate together w main thread
self._logger = ctx._logger
self._logger.info("Results")
self._util_cmd = ctx.util_cmd
self._resultq = ctx.resultq
self._json = JSONCommon(ctx)
self._tuuid = TUUIDCommon(ctx)
self._failfast_ev = ctx.failfast_ev
self._startfast_br = ctx.startfast_br
def _log_finish_event(self, resultnt: ResultNT) -> None:
nowdt = self._tuuid.get_tza_utcdt()
finishtd = nowdt - resultnt.startdt
message = (f"{self._util_cmd} finish:: tuuid: {resultnt.tuuid}; "
f"finishtd: {finishtd}; destdirp: {resultnt.destdirp}; "
f"worker: {resultnt.worker}")
updated = {
"event_type": "detail",
"event": "finish",
"util_cmd": self._util_cmd,
"finishtd": finishtd,
"message": message,
}
msgd = resultnt._asdict()
msgd.update(updated)
self._logger.info(self._json.jsonsafe(msgd))
def run(self) -> None:
"""
Run in a background thread.
"""
self._startfast_br.wait() # blocks until all threads are ready
self._logger.info("Results running")
try:
while True:
resultnt = self._resultq.get() # blocks
if isinstance(resultnt, ResultNT):
self._log_finish_event(resultnt)
else: # shouldn't happen
self._logger.error("Invalid resultq object")
self._failfast_ev.set()
except Exception as e:
t = traceback.format_exc()
msg = f"results thread failed: {e}\n{t}"
self._logger.error(msg)
self._failfast_ev.set()
class WorkerDemo(Thread):
"""
Transform image and queue work results.
NOTE: the Worker is where the bulk of the I/O- or CPU-related work occurs.
In this example, the worker is primarily transforming images, but tasks such
as API calls or heavy computations would fit well here.
"""
def __init__(self, ctx: ContextBase, worker: int) -> None:
"""
Args:
ctx: The Context object.
worker: The worker index.
"""
super().__init__(daemon=True) # worker terminates if parent does
self._logger = ctx._logger
self._logger.info(f"worker: {worker}")
self._worker = worker
self._workq = ctx.workq
self._resultq = ctx.resultq
self._tuuid = TUUIDCommon(ctx)
self._transforms = TransformsDemo(ctx)
self._timeout = ctx.timeout
self._modified_dirp = ctx.MODIFIED_DIRP
self._original_dirp = ctx.ORIGINAL_DIRP
self._rejected_dirp = ctx.REJECTED_DIRP
self._failfast_ev = ctx.failfast_ev
self._startfast_br = ctx.startfast_br
def _save_original(self, filep: Path) -> None:
with Image(filename=str(filep)) as img:
copyp = self._original_dirp / filep.name
img.save(filename=str(copyp))
def _process_work(self, worknt: WorkNT) -> None:
beginworktd = self._tuuid.get_tza_utcdt() - worknt.startdt
filep = worknt.filep
transforms: List[str] = []
if filep.exists():
if worknt.valid:
self._save_original(filep)
transforms = self._transforms.run_transforms(str(filep))
destdirp = self._modified_dirp
else:
destdirp = self._rejected_dirp
new_name = destdirp / filep.name
filep.rename(new_name) # move to destination directory
else:
msg = ("filepath does not currently exist; "
f"worker: {self._worker}; worknt: {worknt}")
self._logger.warning(msg)
updated = {
"worker": self._worker,
"beginworktd": beginworktd,
"endworktd": self._tuuid.get_tza_utcdt() - worknt.startdt,
"destdirp": destdirp,
"transforms": transforms,
}
resultd = worknt._asdict()
resultd.update(updated)
resultnt = ResultNT(**resultd)
self._resultq.put(resultnt)
def _kill_switch(self, worknt: WorkNT) -> None:
msg = f"worker hit {self._timeout} sec timeout processing worknt: {worknt}"
self._logger.error(msg)
self._failfast_ev.set()
def run(self) -> None:
"""
Start the thread.
"""
self._startfast_br.wait() # blocks until all threads are ready
self._logger.info(f"Worker: {self._worker} running")
try:
while True:
worknt = None # make sure it is defined for use in except
worknt = self._workq.get() # blocks
ks = Timer(self._timeout, self._kill_switch, args=[worknt])
ks.start()
self._process_work(worknt)
ks.cancel()
except Exception as e:
t = traceback.format_exc()
msg = f"worker run failed: {e}\nworknt: {worknt}\n{t}"
self._logger.error(msg)
self._failfast_ev.set()
class IncomingDemo(Thread):
"""
Poll a directory and queue files to worker bkgd threads.
NOTE: we often use this Incoming model with pulling a Pub/Sub subscription
asynchronously. The polling of a directory that you see here can be swapped
out with other queue, stream, and pub/sub mechanisms.
"""
def __init__(self, ctx: ContextBase) -> None:
"""
Args:
ctx: The Context object.
"""
super().__init__(daemon=True) # terminate together w main thread
self._logger = ctx._logger
self._logger.info("Incoming")
self._workq = ctx.workq
self._util_cmd = ctx.util_cmd
self._timeout = ctx.timeout
self._json = JSONCommon(ctx)
self._tuuid = TUUIDCommon(ctx)
self._check_interval_secs = ctx.check_interval_secs
self._lastcheck_time = float(0) # init to zero
self._incoming_dirp = ctx.INCOMING_DIRP
self._valid_extensions = ctx.VALID_EXTENSIONS
self._failfast_ev = ctx.failfast_ev
self._startfast_br = ctx.startfast_br
def _log_event(self, blobd: Dict[str, Any], event: str) -> None:
tuuid = blobd["tuuid"]
filep = blobd["filep"]
msgd = blobd.copy()
update = {
"event_type":
"detail",
"event":
event,
"util_cmd":
self._util_cmd,
"message":
(f"{self._util_cmd} {event}:: tuuid: {tuuid}; filep: {filep}", ),
}
msgd.update(update)
self._logger.info(self._json.jsonsafe(msgd))
def _submit_work(self, filepd: Dict[str, Any]) -> None:
startdt = filepd["startdt"]
filepd["queuedtd"] = self._tuuid.get_tza_utcdt() - startdt
self._workq.put(WorkNT(**filepd))
def _valid_filep(self, filep: Path) -> bool:
msgs = []
extension = filep.suffix.upper()
size = filep.stat().st_size
if extension not in self._valid_extensions:
msgs.append(f"invalid extension: {extension}")
if size == 0:
msgs.append("zero-length file")
if msgs:
msg = "; ".join(msgs)
msg += f"; filep: {filep}; "
self._logger.warning(msg)
return False
return True
def _process_filep(self, filep: Path) -> None:
tuuid = self._tuuid.get_tuuid()
valid = self._valid_filep(filep)
startdt = self._tuuid.extract_datetime(tuuid)
filepd = {
"tuuid": tuuid,
"filep": filep,
"valid": valid,
"startdt": startdt
}
self._submit_work(filepd)
self._log_event(filepd, "start")
def _check_incoming(self) -> None:
for filep in self._incoming_dirp.iterdir(
): # does not handle nested dirs
if filep.stat().st_ctime > self._lastcheck_time:
self._process_filep(filep)
self._lastcheck_time = time.time()
def _kill_switch(self) -> None:
msg = f"incoming hit {self._timeout} sec timeout"
self._logger.critical(msg)
self._failfast_ev.set()
def run(self) -> None:
"""
Run the thread.
Background thread of parent process.
"""
self._startfast_br.wait() # blocks until all threads are ready
self._logger.info("Incoming running")
try:
while True:
ks = Timer(self._timeout, self._kill_switch)
ks.start()
self._check_incoming()
ks.cancel()
time.sleep(self._check_interval_secs)
except Exception as e:
t = traceback.format_exc()
msg = f"incoming thread failed: {e}\n{t}"
self._logger.error(msg)
self._failfast_ev.set()