def run_task_manifest_on_cluster(path):
"""
Run the Task on the queue (of possible)
:param path:
:return:
"""
output_dir = os.path.dirname(path)
os.chdir(output_dir)
rt = RunnableTask.from_manifest_json(path)
# this needs to be updated to have explicit paths to stderr, stdout
rcode, err_msg, run_time = run_task_on_cluster(rt, path, output_dir, True)
cstderr = os.path.join(output_dir, "cluster.stderr")
stderr = os.path.join(output_dir, "stderr")
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
# Need to update the run_task_on_cluster
emsg = ""
if rcode != 0:
# try to provide a hint of the exception from the stderr
detail_msg = _extract_last_nlines(stderr)
cdetails_msg = _extract_last_nlines(cstderr)
emsg = "{i} Failed with exit code {r} {c}\n{x}".format(r=rcode, i=rt.task.task_id, x=detail_msg, c=cdetails_msg)
return state, err_msg, run_time
def run_task_manifest_on_cluster(path):
"""
Run the Task on the queue (of possible)
:param path:
:return:
"""
output_dir = os.path.dirname(path)
os.chdir(output_dir)
rt = RunnableTask.from_manifest_json(path)
# this needs to be updated to have explicit paths to stderr, stdout
rcode, err_msg, run_time = run_task_on_cluster(rt, path, output_dir, True)
cstderr = os.path.join(output_dir, "cluster.stderr")
stderr = os.path.join(output_dir, "stderr")
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
# Need to update the run_task_on_cluster
emsg = ""
if rcode != 0:
# try to provide a hint of the exception from the stderr
detail_msg = _extract_last_nlines(stderr)
cdetails_msg = _extract_last_nlines(cstderr)
emsg = "{i} Failed with exit code {r} {c}\n{x}".format(r=rcode, i=rt.task.task_id, x=detail_msg, c=cdetails_msg)
return state, err_msg, run_time
def run_task_manifest(path):
output_dir = os.path.dirname(path)
os.chdir(output_dir)
stderr = os.path.join(output_dir, 'stderr')
stdout = os.path.join(output_dir, 'stdout')
try:
rt = RunnableTask.from_manifest_json(path)
except KeyError:
emsg = "Unable to deserialize RunnableTask from manifest {p}".format(
p=path)
log.error(emsg)
raise
rcode, err_msg, run_time = run_task(rt, output_dir, stdout, stderr, True)
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
emsg = ""
if rcode != 0:
# try to provide a hint of the exception from the stderr
detail_msg = _extract_last_nlines(stderr)
emsg = "{i} Failed with exit code {r} {x}".format(r=rcode,
i=rt.task.task_id,
x=detail_msg)
return state, err_msg, run_time
def _args_run_task_manifest(args):
output_dir = os.getcwd() if args.output_dir is None else args.output_dir
task_manifest_path = args.task_manifest
rt = RunnableTask.from_manifest_json(task_manifest_path)
rcode, _ = run_task(rt, output_dir, args.task_stdout, args.task_stderr, args.debug)
return rcode
def _args_run_task_manifest(args):
output_dir = os.getcwd() if args.output_dir is None else args.output_dir
task_manifest_path = args.task_manifest
rt = RunnableTask.from_manifest_json(task_manifest_path)
rcode, _ = run_task(rt, output_dir, args.task_stdout, args.task_stderr,
args.debug)
return rcode
def _args_run_task_manifest(args):
output_dir = os.getcwd() if args.output_dir is None else args.output_dir
task_manifest_path = args.task_manifest
log.info("Loading runnable-task from {f}".format(f=task_manifest_path))
rt = RunnableTask.from_manifest_json(task_manifest_path)
log.info("loaded runnable-task")
# (exit code, run_time_sec) =
rcode, err_msg, _ = run_task(rt, output_dir, args.task_stdout, args.task_stderr, args.debug)
return rcode
def _args_run_task_manifest(args):
output_dir = os.getcwd() if args.output_dir is None else args.output_dir
task_manifest_path = args.task_manifest
log.info("Loading runnable-task from {f}".format(f=task_manifest_path))
rt = RunnableTask.from_manifest_json(task_manifest_path)
log.info("loaded runnable-task")
# (exit code, run_time_sec) =
rcode, err_msg, _ = run_task(rt, output_dir, args.task_stdout, args.task_stderr, args.debug)
return rcode
def run_task_manifest_on_cluster(path):
"""
Run the Task on the queue (of possible)
:param path:
:return:
"""
output_dir = os.path.dirname(path)
rt = RunnableTask.from_manifest_json(path)
rcode, run_time = run_task_on_cluster(rt, path, output_dir, True)
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
msg = "{r} failed".format(r=rt) if rcode != 0 else ""
return state, msg, run_time
def run_task_manifest_on_cluster(path):
"""
Run the Task on the queue (of possible)
:param path:
:return:
"""
output_dir = os.path.dirname(path)
rt = RunnableTask.from_manifest_json(path)
rcode, run_time = run_task_on_cluster(rt, path, output_dir, True)
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
msg = "{r} failed".format(r=rt) if rcode != 0 else ""
return state, msg, run_time
def run_task_manifest_on_cluster(path):
"""
Run the Task on the queue (of possible)
:param path:
:return:
"""
output_dir = os.path.dirname(path)
os.chdir(output_dir)
rt = RunnableTask.from_manifest_json(path)
# this needs to be updated to have explicit paths to stderr, stdout
rcode, err_msg, run_time = run_task_on_cluster(rt, path, output_dir, True)
state = TaskStates.from_int(rcode)
return state, err_msg, run_time
def run_task_manifest_on_cluster(path):
"""
Run the Task on the queue (of possible)
:param path:
:return:
"""
output_dir = os.path.dirname(path)
os.chdir(output_dir)
rt = RunnableTask.from_manifest_json(path)
# this needs to be updated to have explicit paths to stderr, stdout
rcode, err_msg, run_time = run_task_on_cluster(rt, path, output_dir, True)
state = TaskStates.from_int(rcode)
return state, err_msg, run_time
def run_task_manifest(path):
output_dir = os.path.dirname(path)
stderr = os.path.join(output_dir, 'stderr')
stdout = os.path.join(output_dir, 'stdout')
try:
rt = RunnableTask.from_manifest_json(path)
except KeyError:
emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
log.error(emsg)
raise
rcode, run_time = run_task(rt, output_dir, stdout, stderr, True)
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
msg = "" if rcode == 0 else "Failed with exit code {r}".format(r=rcode)
return state, msg, run_time
def run_task_manifest(path):
output_dir = os.path.dirname(path)
os.chdir(output_dir)
stderr = os.path.join(output_dir, 'stderr')
stdout = os.path.join(output_dir, 'stdout')
try:
rt = RunnableTask.from_manifest_json(path)
except KeyError:
emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
log.error(emsg)
raise
# blocking call
rcode, err_msg, run_time = run_task(rt, output_dir, stdout, stderr, True)
state = TaskStates.from_int(rcode)
return state, err_msg, run_time
def run_task_manifest(path):
output_dir = os.path.dirname(path)
stderr = os.path.join(output_dir, 'stderr')
stdout = os.path.join(output_dir, 'stdout')
try:
rt = RunnableTask.from_manifest_json(path)
except KeyError:
emsg = "Unable to deserialize RunnableTask from manifest {p}".format(
p=path)
log.error(emsg)
raise
rcode, run_time = run_task(rt, output_dir, stdout, stderr, True)
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
msg = "" if rcode == 0 else "Failed with exit code {r}".format(r=rcode)
return state, msg, run_time
def run_task_manifest(path):
output_dir = os.path.dirname(path)
os.chdir(output_dir)
stderr = os.path.join(output_dir, 'stderr')
stdout = os.path.join(output_dir, 'stdout')
try:
rt = RunnableTask.from_manifest_json(path)
except KeyError:
emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
log.error(emsg)
raise
# blocking call
rcode, err_msg, run_time = run_task(rt, output_dir, stdout, stderr, True)
state = TaskStates.from_int(rcode)
return state, err_msg, run_time
def run_task_manifest(path):
output_dir = os.path.dirname(path)
os.chdir(output_dir)
stderr = os.path.join(output_dir, 'stderr')
stdout = os.path.join(output_dir, 'stdout')
try:
rt = RunnableTask.from_manifest_json(path)
except KeyError:
emsg = "Unable to deserialize RunnableTask from manifest {p}".format(p=path)
log.error(emsg)
raise
rcode, err_msg, run_time = run_task(rt, output_dir, stdout, stderr, True)
state = TaskStates.SUCCESSFUL if rcode == 0 else TaskStates.FAILED
emsg = ""
if rcode != 0:
# try to provide a hint of the exception from the stderr
detail_msg = _extract_last_nlines(stderr)
emsg = "{i} Failed with exit code {r} {x}".format(r=rcode, i=rt.task.task_id, x=detail_msg)
return state, err_msg, run_time
def loadRtcs(cls):
cls.tasks_dir = op.join(cls.job_dir, "tasks")
cls.resolved_tool_contracts = []
cls.runnable_tasks = []
tasks_rpt = op.join(cls.job_dir, "workflow", "report-tasks.json")
rpt = load_report_from_json(tasks_rpt)
table = {t.id: t for t in rpt.tables}['tasks']
tasks = {c.id: c.values for c in table.columns}['task_id']
for task_id_str in tasks:
fields = task_id_str.split()
task_name = fields[1].split(":")[-1]
task_dir = op.join(cls.tasks_dir, task_name)
if not op.isdir(task_dir):
continue
task_id, job_id = task_name.split("-")
rtc_json = op.join(task_dir, "resolved-tool-contract.json")
if not op.isfile(rtc_json):
log.warn("Can't find %s" % rtc_json)
continue
rtc = load_resolved_tool_contract_from(rtc_json)
cls.resolved_tool_contracts.append(rtc)
rt_json = op.join(task_dir, "runnable-task.json")
rt = RunnableTask.from_manifest_json(rt_json)
cls.runnable_tasks.append(rt)
def _args_to_cmd(args):
return run_to_cmd(RunnableTask.from_manifest_json(args.task_manifest))
def _args_to_cmd(args):
return run_to_cmd(RunnableTask.from_manifest_json(args.task_manifest))
def _args_pprint_task_manifest(args):
return pprint_task_manifest(RunnableTask.from_manifest_json(args.task_manifest))
def test_task_manifest_serialization(self):
path = os.path.join(TEST_DATA_DIR, 'task-manifest.json')
r = RunnableTask.from_manifest_json(path)
self.assertIsInstance(r, RunnableTask)
def validate_file_and_load_manifest(path):
rt = RunnableTask.from_manifest_json(validate_file(path))
# if we got here everything is valid
return path
def validate_file_and_load_manifest(path):
rt = RunnableTask.from_manifest_json(validate_file(path))
# if we got here everything is valid
return path
def _args_pprint_task_manifest(args):
return pprint_task_manifest(
RunnableTask.from_manifest_json(args.task_manifest))
def test_task_manifest_serialization(self):
path = os.path.join(TEST_DATA_DIR, 'task-manifest.json')
r = RunnableTask.from_manifest_json(path)
self.assertIsInstance(r, RunnableTask)