def register_report(name, values, template=None, required=None, update_frequency=300):
report = graph.OutputReport(output_path=name,
report_values=values,
report_template=template,
required=required,
update_frequency=update_frequency)
sis_graph.add_target(report)
return report
def run(obj: Any, quiet: bool = False):
"""
Run and setup all jobs that are contained inside object and all jobs that are necessary.
:param obj:
:param quiet: Do not forward job output do stdout
:return:
"""
def run_helper(job):
"""
Helper function which takes a job and runs it task until it's finished
:param job: Job to run
:return:
"""
assert job._sis_runnable()
if not job._sis_finished():
logging.info("Run Job: %s" % job)
job._sis_setup_directory()
for task in job._sis_tasks():
for task_id in task.task_ids():
if not task.finished(task_id):
if len(job._sis_tasks()) > 1 or len(
task.task_ids()) > 1:
logging.info("Run Task: %s %s %s" %
(job, task.name(), task_id))
log_file = task.path(gs.JOB_LOG, task_id)
env = os.environ.copy()
env.update(gs.ENVIRONMENT_SETTINGS)
call = " ".join(task.get_worker_call(task_id))
if quiet:
call += ' --redirect_output'
else:
call += ' 2>&1 > %s' % log_file
subprocess.check_call(call, shell=True, env=env)
assert task.finished(
task_id), "Failed to run task %s %s %s" % (
job, task.name(), task_id)
# Create fresh graph and add object as report since a report can handle all kinds of objects.
temp_graph = graph.SISGraph()
temp_graph.add_target(
graph.OutputReport(output_path='tmp',
report_values=obj,
report_template=None,
required=None,
update_frequency=0))
# Update SIS_COMMAND
import sys
gs.SIS_COMMAND = [sys.executable, '-m', 'sisyphus']
gs.SKIP_IS_FINISHED_TIMEOUT = True
def get_jobs():
""" Helper function to get all relevant jobs"""
filter_list = (gs.STATE_WAITING, gs.STATE_RUNNABLE,
gs.STATE_INTERRUPTED, gs.STATE_ERROR)
return {
k: v
for k, v in temp_graph.get_jobs_by_status(
skip_finished=True).items() if k in filter_list
}
jobs = get_jobs()
# Iterate over all runnable jobs until it's done
while jobs:
# Collect all jobs that can be run
todo_list = jobs.get(gs.STATE_RUNNABLE, set())
todo_list.update(jobs.get(gs.STATE_INTERRUPTED, set()))
# Stop loop if no jobs can be run
if not todo_list:
logging.error(
"Can not finish computation of %s some jobs are blocking" %
obj)
for k, v in temp_graph.get_jobs_by_status(
skip_finished=True).items():
if k != gs.STATE_INPUT_PATH:
logging.error("Jobs in state %s are: %s" % (k, v))
raise BlockedWorkflow(
"Can not finish computation of %s some jobs are blocking" %
obj)
# Actually run the jobs
for job in todo_list:
run_helper(job)
jobs = get_jobs()
gs.SKIP_IS_FINISHED_TIMEOUT = False
def run(obj):
"""
Run and setup all jobs that are contained inside object and all jobs that are necessary.
:param obj:
:return:
"""
def run_helper(job):
"""
Helper function which takes a job and runs it task until it's finished
:param job: Job to run
:return:
"""
assert job._sis_runnable()
if not job._sis_finished():
print()
logging.info("Run Job: %s" % job)
job._sis_setup_directory()
for task in job._sis_tasks():
for task_id in task.task_ids():
#print(job, task, task_id)
if not task.finished(task_id):
print()
logging.info("Run Task: %s %s %s" % (job, task.name(), task_id))
log_file = task.path(gs.JOB_LOG, task_id)
env = os.environ.copy()
env.update(gs.ENVIRONMENT_SETTINGS)
subprocess.check_call(" ".join(task.get_worker_call(task_id)) + ' 2>&1 | tee %s' % log_file,
shell=True, env=env)
# Create fresh graph and add object as report since a report can handle all kinds of objects.
temp_graph = graph.SISGraph()
temp_graph.add_target(graph.OutputReport(output_path='tmp',
report_values=obj,
report_template=None,
required=None,
update_frequency=0))
# Update SIS_COMMAND
import sys
gs.SIS_COMMAND = [sys.executable, '-m', 'sisyphus']
def get_jobs():
""" Helper function to get all relevant jobs"""
return {k: v for k, v in temp_graph.get_jobs_by_status(skip_finished=True).items() if k in [gs.STATE_WAITING,
gs.STATE_RUNNABLE,
gs.STATE_INTERRUPTED,
gs.STATE_ERROR]}
jobs = get_jobs()
# Iterate over all runnable jobs until it's done
while jobs:
# Collect all jobs that can be run
todo_list = jobs.get(gs.STATE_RUNNABLE, set())
todo_list.update(jobs.get(gs.STATE_INTERRUPTED, set()))
# Stop loop if no jobs can be run
if not todo_list:
logging.error(f"Can not finish computation of {obj} some jobs are blockeding")
for k, v in temp_graph.get_jobs_by_status(skip_finished=True):
logging.error(f"Jobs in state {k} are: {v}")
break
# Actually run the jobs
for job in todo_list:
run_helper(job)
jobs = get_jobs()
