def jsonSave(obj: object, dest: Any) -> None:
'''
Serilize an object composed of List and Dict into file specified by
path or io wrapper.
'''
if isinstance(dest, str):
gz, zst = dest.endswith('.gz'), dest.endswith('.zst')
if zst:
with open(dest, 'wb') as f:
f.write(zstd.dumps(json.dumps(obj).encode()))
else:
with (gzip.open(dest, 'wb') if gz else open(dest, 'w')) as f:
f.write(json.dumps(obj).encode() if gz else json.dumps(obj))
elif isinstance(dest, _io.TextIOWrapper) or isinstance(dest, _io.BufferedWriter):
json.dump(obj, dest)
else:
raise TypeError(f'Unknown destination type {type(dest)}')
def tasqueWorker(
db: db.tqDB,
log: object,
task: defs.Task,
):
'''
worker function for processing a Task.
(id, pid, cwd, cmd, retval, stime, etime, pri, rsc)
'''
pid = os.getpid()
# update database before working
sql = f"update tq set pid = {pid}, stime = {time.time()} where (id = {task.id})"
log.info(f'worker[{os.getpid()}]: SQL(pre-task) -- {sql}')
db(sql)
# trying to start task
try:
# change directory, fork and execute the task.
os.chdir(task.cwd)
cmd = shlex.split(task.cmd)
log.info(f'worker[{os.getpid()}]: Command: {str(cmd)}')
proc = subprocess.Popen(cmd,
shell=False,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=task.cwd)
# override daemon's sigaction handler
def sigterm_handler(signo, frame):
log.info(
f'worker[{os.getpid()}] recieved SIGTERM. Gracefully pulling down task process...'
)
proc.kill()
os._exit(0)
signal.signal(signal.SIGTERM, sigterm_handler)
# override daemon's atexit action
atexit.register(lambda: None)
# If nothing goes wrong, we'll get the content of stdout and stderr
stdout, _ = proc.communicate()
retval = proc.returncode
except FileNotFoundError as e:
log.error(f'worker[{os.getpid()}]: {str(e)}')
stdout, retval = '', '', -1
except Exception as e:
log.error(f'worker[{os.getpid()}]: {str(e)}')
stdout, retval = '', '', -1
finally:
log.info(
f'worker[{os.getpid()}]: subprocess.Popen() successfully returned.'
)
# write the stdout (stderr was redirected here)
os.chdir(defs.TASQUE_DIR)
timestamp = time.strftime('%Y%m%d.%H%M%S')
if len(stdout) > 0:
with open(f'tq_id-{task.id}_{timestamp}.stdout.zst', 'wb') as f:
f.write(zstd.dumps(stdout))
# update database after finishing the task
sql = f"update tq set retval = {retval}, etime = {time.time()}, pid = null where (id = {task.id})"
log.info(f'worker[{os.getpid()}]: SQL(post-task) -- {sql}')
db(sql)
# END
log.info(f'worker[{os.getpid()}]: end gracefully.')
def test_random(self): DATA = os.urandom(128 * 1024) # Read 128kb self.assertEqual(DATA, zstd.loads(zstd.dumps(DATA)))
def helper_compression_random(self):
DATA = os.urandom(128 * 1024) # Read 128kb
self.assertEqual(DATA, zstd.loads(zstd.dumps(DATA)))
def _tqWorker(dbpath: str, task: tuple) -> None:
'''
Process a task.
(id, pid, cwd, cmd, retval, stime, etime, pri, rsc)
'''
task = (x if x is not None else 'null' for x in task)
id_, pid, cwd, cmd, retval, stime, etime, pri, rsc = task
pid = os.getpid()
# update database before working
sql = f"update tq set pid = {pid}, stime = {time.time()}" \
+ f" where (id = {id_})"
log.info(f'Worker[{os.getpid()}]: SQL update -- {sql}')
dbExec(dbpath, sql)
try:
# change directory, fork and execute the task.
os.chdir(cwd)
proc = subprocess.Popen(shlex.split(cmd),
shell=False,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd)
# override TQD's sigaction handler
def sigterm_handler(signo, frame):
log.info(
f'Worker[{os.getpid()}] recieved SIGTERM. Gracefully pulling down task process...'
)
proc.kill()
os._exit(0)
signal.signal(signal.SIGTERM, sigterm_handler)
# If nothing goes wrong, we'll get the content of stdout and stderr
tqout, tqerr = proc.communicate()
retval = proc.returncode
except FileNotFoundError as e:
log.error(f'Worker[{os.getpid()}]: {str(e)}')
tqout, tqerr, retval = '', '', -1
except Exception as e:
log.error(f'Worker[{os.getpid()}]: {str(e)}')
tqout, tqerr, retval = '', '', -1
finally:
log.info(
f'Worker[{os.getpid()}]: subprocess.Popen() successfully returned without exception.'
)
os.chdir(cwd)
timestamp = time.strftime('%Y%m%d.%H%M%S')
if len(tqout) > 0:
with open(f'tq_id-{id_}_{timestamp}.stdout.zst', 'wb') as f:
f.write(zstd.dumps(tqout))
if len(tqerr) > 0:
with open(f'tq_id-{id_}_{timestamp}.stderr.zst', 'wb') as f:
f.write(zstd.dumps(tqerr))
# update database after finishing the task
sql = f"update tq set retval = {retval}, etime = {time.time()}," \
+ f"pid = null where (id = {id_})"
log.info(f'Worker[{os.getpid()}]: SQL update -- {sql}')
dbExec(dbpath, sql)
# don't remove pidfile! i.e. don't trigger atexit().
os._exit(0)