def register_report(name, values, template=None, required=None, update_frequency=300): report = graph.OutputReport(output_path=name, report_values=values, report_template=template, required=required, update_frequency=update_frequency) sis_graph.add_target(report) return report
def run(obj: Any, quiet: bool = False): """ Run and setup all jobs that are contained inside object and all jobs that are necessary. :param obj: :param quiet: Do not forward job output do stdout :return: """ def run_helper(job): """ Helper function which takes a job and runs it task until it's finished :param job: Job to run :return: """ assert job._sis_runnable() if not job._sis_finished(): logging.info("Run Job: %s" % job) job._sis_setup_directory() for task in job._sis_tasks(): for task_id in task.task_ids(): if not task.finished(task_id): if len(job._sis_tasks()) > 1 or len( task.task_ids()) > 1: logging.info("Run Task: %s %s %s" % (job, task.name(), task_id)) log_file = task.path(gs.JOB_LOG, task_id) env = os.environ.copy() env.update(gs.ENVIRONMENT_SETTINGS) call = " ".join(task.get_worker_call(task_id)) if quiet: call += ' --redirect_output' else: call += ' 2>&1 > %s' % log_file subprocess.check_call(call, shell=True, env=env) assert task.finished( task_id), "Failed to run task %s %s %s" % ( job, task.name(), task_id) # Create fresh graph and add object as report since a report can handle all kinds of objects. temp_graph = graph.SISGraph() temp_graph.add_target( graph.OutputReport(output_path='tmp', report_values=obj, report_template=None, required=None, update_frequency=0)) # Update SIS_COMMAND import sys gs.SIS_COMMAND = [sys.executable, '-m', 'sisyphus'] gs.SKIP_IS_FINISHED_TIMEOUT = True def get_jobs(): """ Helper function to get all relevant jobs""" filter_list = (gs.STATE_WAITING, gs.STATE_RUNNABLE, gs.STATE_INTERRUPTED, gs.STATE_ERROR) return { k: v for k, v in temp_graph.get_jobs_by_status( skip_finished=True).items() if k in filter_list } jobs = get_jobs() # Iterate over all runnable jobs until it's done while jobs: # Collect all jobs that can be run todo_list = jobs.get(gs.STATE_RUNNABLE, set()) todo_list.update(jobs.get(gs.STATE_INTERRUPTED, set())) # Stop loop if no jobs can be run if not todo_list: logging.error( "Can not finish computation of %s some jobs are blocking" % obj) for k, v in temp_graph.get_jobs_by_status( skip_finished=True).items(): if k != gs.STATE_INPUT_PATH: logging.error("Jobs in state %s are: %s" % (k, v)) raise BlockedWorkflow( "Can not finish computation of %s some jobs are blocking" % obj) # Actually run the jobs for job in todo_list: run_helper(job) jobs = get_jobs() gs.SKIP_IS_FINISHED_TIMEOUT = False
def run(obj): """ Run and setup all jobs that are contained inside object and all jobs that are necessary. :param obj: :return: """ def run_helper(job): """ Helper function which takes a job and runs it task until it's finished :param job: Job to run :return: """ assert job._sis_runnable() if not job._sis_finished(): print() logging.info("Run Job: %s" % job) job._sis_setup_directory() for task in job._sis_tasks(): for task_id in task.task_ids(): #print(job, task, task_id) if not task.finished(task_id): print() logging.info("Run Task: %s %s %s" % (job, task.name(), task_id)) log_file = task.path(gs.JOB_LOG, task_id) env = os.environ.copy() env.update(gs.ENVIRONMENT_SETTINGS) subprocess.check_call(" ".join(task.get_worker_call(task_id)) + ' 2>&1 | tee %s' % log_file, shell=True, env=env) # Create fresh graph and add object as report since a report can handle all kinds of objects. temp_graph = graph.SISGraph() temp_graph.add_target(graph.OutputReport(output_path='tmp', report_values=obj, report_template=None, required=None, update_frequency=0)) # Update SIS_COMMAND import sys gs.SIS_COMMAND = [sys.executable, '-m', 'sisyphus'] def get_jobs(): """ Helper function to get all relevant jobs""" return {k: v for k, v in temp_graph.get_jobs_by_status(skip_finished=True).items() if k in [gs.STATE_WAITING, gs.STATE_RUNNABLE, gs.STATE_INTERRUPTED, gs.STATE_ERROR]} jobs = get_jobs() # Iterate over all runnable jobs until it's done while jobs: # Collect all jobs that can be run todo_list = jobs.get(gs.STATE_RUNNABLE, set()) todo_list.update(jobs.get(gs.STATE_INTERRUPTED, set())) # Stop loop if no jobs can be run if not todo_list: logging.error(f"Can not finish computation of {obj} some jobs are blockeding") for k, v in temp_graph.get_jobs_by_status(skip_finished=True): logging.error(f"Jobs in state {k} are: {v}") break # Actually run the jobs for job in todo_list: run_helper(job) jobs = get_jobs()