def insert_tasks(PL, task_file): pipeline = Pipeline(PL.name, PL.log_dir) logging.debug("Pipeline is: {}".format(pipeline)) task_list = PL.prepare_managed_tasks() logging.debug("Task list is: {}".format([x['name'] for x in task_list])) # we need to be able to translate the dependencies as stored in the task # list (list of other task names that a particular task depends on) # into a list of Job object references that have already been added to the # session. We will build up a dictionary of task['name'] : Job as we # insert them deps_to_job = {} print(" Inserting tasks into {}".format(task_file)) logging.info("Inserting tasks into {}".format(task_file)) try: for task in task_list: print(" -> {}".format(task['name'])) try: dependencies = [deps_to_job[d] for d in task['dependencies']] except KeyError as e: logging.exception("Key error processing dependencies") msg = "Task {} depends on a task that hasn't been been " \ "processed ({}). Check your Pipeline XML".format( task['name'], e.args[0]) raise Exception(msg) job = Job(pipeline, task['name'], task['threads'], task['stdout_path'], task['stderr_path'], task['script_path'], task['epilogue_path'], task['mem'], task['email_list'], task['mail_options'], task['batch_env'], dependencies, task['queue'], task['walltime']) deps_to_job[task['name']] = job logging.debug("Adding job {} (log dir: {}) to session".format( job.job_name, job.pipeline.log_directory)) Session.add(job) except Exception as e: logging.exception("Error inserting tasks into database") print("Error inserting tasks into database: {}".format(e), file=sys.stderr) sys.exit(6) # only commit the session if we were able to add all the jobs to the session # without catching an Exception Session.commit() logging.info(" {} tasks have been inserted into task file {}; " "(log dir: {})".format(len(task_list), task_file, PL.log_dir)) return len(task_list)