def create_task_base(self, wf: Workflow, job: PreparedJob):
forbiddenids = set()
if job.store_in_central_db:
try:
with self.with_cursor() as cursor:
forbiddenids = set(
t[0] for t in cursor.execute("SELECT id FROM tasks").fetchall()
)
except sqlite3.OperationalError as e:
if "no such column: id" in repr(e):
from shutil import move
dt = datetime.utcnow()
np = f"{job.db_path}.original-{dt.strftime('%Y%m%d')}"
Logger.warn(f"Moving old janis-db to '{np}'")
move(job.db_path, np)
self._taskDB = None
return self.create_task_base(wf=wf, job=job)
raise
submission_id = generate_new_id(forbiddenids)
output_dir = fully_qualify_filename(job.output_dir)
if not job.execution_dir:
job.execution_dir = os.path.join(output_dir, "janis")
Logger.debug(
f"No execution-dir was provided, constructed one from the output-dir: {job.execution_dir}"
)
job.execution_dir = fully_qualify_filename(job.execution_dir)
Logger.info(
f"Starting task with id = '{submission_id}' | output dir: {job.output_dir} | execution dir: {job.execution_dir}"
)
row = TaskRow(
submission_id, execution_dir=job.execution_dir, output_dir=output_dir
)
WorkflowManager.create_dir_structure(job.execution_dir)
if job.store_in_central_db:
self.get_lazy_db_connection().insert_task(row)
else:
Logger.info(
f"Not storing task '{submission_id}' in database. To watch, use: 'janis watch {output_dir}'"
)
if self._connection:
self._connection.commit()
self._connection.close()
self._taskDB = None
self._connection = None
return row
def create_task_base(self, wf: Workflow, outdir=None, store_in_centraldb=True):
config = JanisConfiguration.manager()
"""
If you don't spec
"""
if not outdir and not config.outputdir:
raise Exception(
f"You must specify an output directory (or specify an '{JanisConfiguration.Keys.OutputDir.value}' "
f"in your configuration)"
)
default_outdir = None
if config.outputdir:
default_outdir = os.path.join(config.outputdir, wf.id())
forbiddenids = set()
if store_in_centraldb:
with self.with_cursor() as cursor:
forbiddenids = set(
t[0] for t in cursor.execute("SELECT wid FROM tasks").fetchall()
)
if outdir:
if os.path.exists(outdir):
# this should theoretically scoop through all the ones in the taskDB and
# add them to the forbidden ones, though this might cause more issues for now.
forbiddenids = forbiddenids.union(set(os.listdir(outdir)))
else:
if os.path.exists(default_outdir):
forbiddenids = forbiddenids.union(set(os.listdir(default_outdir)))
wid = generate_new_id(forbiddenids)
task_path = outdir
if not task_path:
od = default_outdir
dt = datetime.now().strftime("%Y%m%d_%H%M%S")
task_path = os.path.join(od, f"{dt}_{wid}/")
task_path = fully_qualify_filename(task_path)
Logger.info(f"Starting task with id = '{wid}'")
row = TaskRow(wid, task_path)
WorkflowManager.create_dir_structure(task_path)
if store_in_centraldb:
self.get_lazy_db_connection().insert_task(row)
else:
Logger.info(
f"Not storing task '{wid}' in database. To watch, use: 'janis watch {task_path}'"
)
if self._connection:
self._connection.commit()
self._connection.close()
self._taskDB = None
self._connection = None
return row