def insert_tasks(PL, task_file):
pipeline = Pipeline(PL.name, PL.log_dir)
logging.debug("Pipeline is: {}".format(pipeline))
task_list = PL.prepare_managed_tasks()
logging.debug("Task list is: {}".format([x['name'] for x in task_list]))
# we need to be able to translate the dependencies as stored in the task
# list (list of other task names that a particular task depends on)
# into a list of Job object references that have already been added to the
# session. We will build up a dictionary of task['name'] : Job as we
# insert them
deps_to_job = {}
print(" Inserting tasks into {}".format(task_file))
logging.info("Inserting tasks into {}".format(task_file))
try:
for task in task_list:
print(" -> {}".format(task['name']))
try:
dependencies = [deps_to_job[d] for d in task['dependencies']]
except KeyError as e:
logging.exception("Key error processing dependencies")
msg = "Task {} depends on a task that hasn't been been " \
"processed ({}). Check your Pipeline XML".format(
task['name'], e.args[0])
raise Exception(msg)
job = Job(pipeline, task['name'], task['threads'],
task['stdout_path'], task['stderr_path'],
task['script_path'], task['epilogue_path'], task['mem'],
task['email_list'], task['mail_options'],
task['batch_env'], dependencies, task['queue'],
task['walltime'])
deps_to_job[task['name']] = job
logging.debug("Adding job {} (log dir: {}) to session".format(
job.job_name, job.pipeline.log_directory))
Session.add(job)
except Exception as e:
logging.exception("Error inserting tasks into database")
print("Error inserting tasks into database: {}".format(e),
file=sys.stderr)
sys.exit(6)
# only commit the session if we were able to add all the jobs to the session
# without catching an Exception
Session.commit()
logging.info(" {} tasks have been inserted into task file {}; "
"(log dir: {})".format(len(task_list), task_file, PL.log_dir))
return len(task_list)
