def test_write_pipeline_preset_json(self):
ofn = tempfile.NamedTemporaryFile(suffix=".json").name
CMD = "pbsmrtpipe show-template-details pbsmrtpipe.pipelines.dev_diagnostic_stress -j {f}".format(f=ofn)
rcode, stdout, stderr, run_time = backticks(CMD)
self.assertEqual(rcode, 0)
with open(ofn) as f:
d = json.load(f)
self.assertEqual(len(d['taskOptions']), 2)
def test_write_workflow_preset_json(self):
from pbsmrtpipe.pb_io import REGISTERED_WORKFLOW_OPTIONS
ofn = tempfile.NamedTemporaryFile(suffix=".json").name
CMD = "pbsmrtpipe show-workflow-options -j {f}".format(f=ofn)
rcode, stdout, stderr, run_time = backticks(CMD)
self.assertEqual(rcode, 0)
with open(ofn) as f:
d = json.load(f)
self.assertEqual(len(d['options']), len(REGISTERED_WORKFLOW_OPTIONS))
def test_write_workflow_preset_json(self):
from pbsmrtpipe.pb_io import REGISTERED_WORKFLOW_OPTIONS
ofn = tempfile.NamedTemporaryFile(suffix=".json").name
CMD = "pbsmrtpipe show-workflow-options -j {f}".format(f=ofn)
rcode, stdout, stderr, run_time = backticks(CMD)
self.assertEqual(rcode, 0)
with open(ofn) as f:
d = json.load(f)
self.assertEqual(len(d['options']), len(REGISTERED_WORKFLOW_OPTIONS))
def test_write_pipeline_preset_json(self):
ofn = tempfile.NamedTemporaryFile(suffix=".json").name
pipeline_id = "pbsmrtpipe.pipelines.dev_diagnostic_stress"
cmd = "pbsmrtpipe show-template-details {i} -j {f}".format(f=ofn, i=pipeline_id)
rcode, stdout, stderr, run_time = backticks(cmd)
fail_msg = "Failed (exit code {e}) to show template details cmd: {c} {m} ".format(i=pipeline_id, e=rcode, c=cmd, m=str(stderr))
self.assertEqual(rcode, 0, fail_msg)
with open(ofn) as f:
d = json.load(f)
self.assertEqual(len(d['taskOptions']), 9)
def _run_testkit_cfg(testkit_cfg, debug=False, misc_opts=Constants.MISC_OPTS):
os.chdir(os.path.dirname(testkit_cfg))
cmd = "{e} --debug {m} {c}".format(c=testkit_cfg, e=Constants.EXE,
m=misc_opts)
rcode, stdout, stderr, run_time = backticks(cmd)
if debug:
log.debug(" ".join([str(i) for i in [cmd, rcode, stdout, stderr]]))
# Returning the butler cfg is a bit odd, but this is necessary for the
# parallelism to work consistently with the serial version
return testkit_cfg, rcode, stdout, stderr, run_time
def _run_testkit_cfg(testkit_cfg, debug=False, misc_opts=""):
os.chdir(os.path.dirname(testkit_cfg))
cmd = "{e} --debug {m} {c}".format(c=testkit_cfg, e=_EXE, m=misc_opts)
rcode, stdout, stderr, run_time = backticks(cmd)
if debug:
log.debug(" ".join([str(i) for i in [cmd, rcode, stdout, stderr]]))
# Returning the butler cfg is a bit odd, but this is necessary for the
# parallelism to work consistently with the serial version
return testkit_cfg, rcode, stdout, stderr, run_time
def test_backticks(self):
exe = 'python --version'
rcode, out, err, run_time = backticks(exe)
self.assertEqual(rcode, 0)
self.assertTrue(out[0].startswith('Python '))
self.assertEqual(err, "")
def test_backticks(self):
exe = 'python --version'
rcode, out, err, run_time = backticks(exe)
self.assertEqual(rcode, 0)
self.assertTrue(out[0].startswith('Python '))
self.assertEqual(err, "")
def _run_testkit_cfg(testkit_cfg,
debug=False,
misc_opts=Constants.MISC_OPTS,
sleep_time=Constants.SLEEP_TIME):
time.sleep(random.randint(1, sleep_time * 2))
os.chdir(os.path.dirname(testkit_cfg))
cmd = "{e} --debug --sleep {t} {m} {c}".format(c=testkit_cfg,
e=Constants.EXE,
m=misc_opts,
t=sleep_time)
rcode, stdout, stderr, run_time = backticks(cmd)
if debug:
log.debug(" ".join([str(i) for i in [cmd, rcode, stdout, stderr]]))
# Returning the butler cfg is a bit odd, but this is necessary for the
# parallelism to work consistently with the serial version
return testkit_cfg, rcode, stdout, stderr, run_time
def _dot_to_image(image_type, dot_file, image_file):
assert image_type.lower() in _SUPPORTED_IMAGE_TYPES
if not os.path.exists(dot_file):
raise IOError("Unable to find {f}".format(f=dot_file))
if which(DOT_EXE) is None:
raise RequiredExeNotFoundError("Unable to find required external exe '{x}'".format(x=DOT_EXE))
cmd_str = "{e} -T{t} {i} -o {o}"
d = dict(e=DOT_EXE, t=image_type, i=dot_file, o=image_file)
cmd = cmd_str.format(**d)
rcode, stdout, stderr, run_time = backticks(cmd)
state = True if rcode == 0 else False
return state
def _dot_to_image(image_type, dot_file, image_file):
assert image_type.lower() in _SUPPORTED_IMAGE_TYPES
if not os.path.exists(dot_file):
raise IOError("Unable to find {f}".format(f=dot_file))
if which(DOT_EXE) is None:
raise RequiredExeNotFoundError(
"Unable to find required external exe '{x}'".format(x=DOT_EXE))
cmd_str = "{e} -T{t} {i} -o {o}"
d = dict(e=DOT_EXE, t=image_type, i=dot_file, o=image_file)
cmd = cmd_str.format(**d)
rcode, stdout, stderr, run_time = backticks(cmd)
state = True if rcode == 0 else False
return state
def _align_to_reference(sequence, reference, job_dir):
alignment_output = tempfile.NamedTemporaryFile(
suffix="_consensus.out").name
cmd = " ".join([
"blasr", sequence, reference,
"--minMatch", "15",
"--maxMatch", "20",
"--fastMaxInterval", "-m", "5",
"--fastSDP", "--aggressiveIntervalCut",
"--advanceExactMatches", "10",
"--out", alignment_output])
errCode, output, errorMessage, run_time = backticks(cmd)
log.debug(cmd)
if errCode is 0:
log.info(output)
elif errCode is 1:
log.error(errorMessage)
return alignment_output
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode):
"""
:param runnable_task:
:param output_dir:
:param debug_mode:
:return:
:type runnable_task: RunnableTask
"""
def _to_p(x_):
return os.path.join(output_dir, x_)
stdout_ = _to_p('stdout')
stderr_ = _to_p('stderr')
if runnable_task.task.is_distributed is False:
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
if runnable_task.cluster is None:
log.warn("No cluster provided. Running task locally.")
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
os.chdir(runnable_task.task.output_dir)
env_json = os.path.join(output_dir, '.cluster-env.json')
IO.write_env_to_json(env_json)
# sloppy API
if isinstance(runnable_task.cluster, ClusterTemplateRender):
render = runnable_task.cluster
else:
ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()]
render = ClusterTemplateRender(ctmpls)
job_id = to_random_job_id(runnable_task.task.task_id)
log.debug("Using job id {i}".format(i=job_id))
qstdout = _to_p('cluster.stdout')
qstderr = _to_p('cluster.stderr')
qshell = _to_p('cluster.sh')
rcmd_shell = _to_p('run.sh')
# This needs to be flattened due to the new RTC layer
# Task Manifest Runner output
stdout = _to_p('stdout')
stderr = _to_p('stderr')
mstdout = _to_p('mstdout')
mstderr = _to_p('mstderr')
with open(qstdout, 'w+') as f:
f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task))
debug_str = " --debug "
exe = _resolve_exe("pbtools-runner")
_d = dict(x=exe,
t=task_manifest_path,
o=stdout,
e=stderr,
d=debug_str,
m=stdout,
n=stderr,
r=output_dir)
cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d)
with open(rcmd_shell, 'w+') as x:
x.write(cmd + "\n")
# Make +x
os.chmod(rcmd_shell, os.stat(rcmd_shell).st_mode | stat.S_IEXEC)
cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc)
log.debug(cluster_cmd)
with open(qshell, 'w') as f:
f.write("#!/bin/bash\n")
f.write(cluster_cmd + "\n")
os.chmod(qshell, os.stat(qshell).st_mode | stat.S_IEXEC)
# host = socket.getfqdn()
host = platform.node()
# so core dumps are written to the job dir
os.chdir(output_dir)
rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell))
if rcode == 0:
err_msg = ""
warn_msg = ""
else:
p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode)
raw_stderr = _extract_last_nlines(stderr)
cluster_raw_stderr = _extract_last_nlines(cstderr)
err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr])
warn_msg = ""
msg_ = "Completed running cluster command in {t:.2f} sec. Exit code {r}".format(r=rcode, t=run_time)
log.info(msg_)
with open(qstdout, 'a') as qf:
qf.write(str(cstdout) + "\n")
qf.write(msg_ + "\n")
with open(qstderr, 'a') as f:
if rcode != 0:
f.write(str(cstderr) + "\n")
f.write(msg_ + "\n")
r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg)
task_report_path = os.path.join(output_dir, 'task-report.json')
msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id)
log.info(msg)
r.write_json(task_report_path)
return rcode, run_time
def was_backticks_successful(cmd):
rcode, _, _, _ = backticks(cmd, )
return rcode == 0
def was_backticks_successful(cmd):
rcode, _, _, _ = backticks(cmd, )
return rcode == 0
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir,
debug_mode):
"""
:param runnable_task:
:param output_dir:
:param debug_mode:
:return:
:type runnable_task: RunnableTask
"""
def _to_p(x_):
return os.path.join(output_dir, x_)
stdout_ = _to_p('stdout')
stderr_ = _to_p('stderr')
if runnable_task.task.is_distributed is False:
return run_task(runnable_task, output_dir, stdout_, stderr_,
debug_mode)
if runnable_task.cluster is None:
log.warn("No cluster provided. Running task locally.")
return run_task(runnable_task, output_dir, stdout_, stderr_,
debug_mode)
env_json = os.path.join(output_dir, 'env.json')
IO.write_env_to_json(env_json)
# sloppy API
if isinstance(runnable_task.cluster, ClusterTemplateRender):
render = runnable_task.cluster
else:
ctmpls = [
ClusterTemplate(name, tmpl)
for name, tmpl in runnable_task.cluster.iteritems()
]
render = ClusterTemplateRender(ctmpls)
job_id = to_random_job_id(runnable_task.task.task_id)
log.debug("Using job id {i}".format(i=job_id))
qstdout = _to_p('cluster.stdout')
qstderr = _to_p('cluster.stderr')
qshell = _to_p('cluster.sh')
rcmd_shell = _to_p('run.sh')
# Task Manifest Runner output
stdout = _to_p('stdout')
stderr = _to_p('stderr')
mstdout = _to_p('mstdout')
mstderr = _to_p('mstderr')
with open(qstdout, 'w+') as f:
f.write("Creating cluster stdout for Job {i} {r}\n".format(
i=job_id, r=runnable_task))
debug_str = " --debug "
_d = dict(t=task_manifest_path,
o=stdout,
e=stderr,
d=debug_str,
m=mstdout,
n=mstderr)
cmd = "pbtools-runner run {d} --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(
**_d)
with open(rcmd_shell, 'w+') as x:
x.write(cmd + "\n")
# Make +x
os.chmod(rcmd_shell, os.stat(rcmd_shell).st_mode | stat.S_IEXEC)
cluster_cmd = render.render('interactive', rcmd_shell, job_id, qstdout,
qstderr, runnable_task.task.nproc)
log.debug(cluster_cmd)
with open(qshell, 'w') as f:
f.write("#!/bin/bash\n")
f.write(cluster_cmd + "\n")
os.chmod(qshell, os.stat(qshell).st_mode | stat.S_IEXEC)
# host = socket.getfqdn()
host = platform.node()
# so core dumps are written to the job dir
os.chdir(output_dir)
rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell))
if rcode == 0:
err_msg = ""
warn_msg = ""
else:
# not sure how to scrape this from the stderr/stdout
err_msg = "task {i} failed".format(i=runnable_task.task.task_id)
warn_msg = ""
msg_ = "Completed running cluster command in {t:.2f} sec. Exit code {r}".format(
r=rcode, t=run_time)
log.info(msg_)
with open(qstdout, 'a') as qf:
qf.write(str(cstdout) + "\n")
qf.write(msg_ + "\n")
with open(qstderr, 'a') as f:
if rcode != 0:
f.write(str(cstderr) + "\n")
f.write(msg_ + "\n")
r = to_task_report(host, runnable_task.task.task_id, run_time, rcode,
err_msg, warn_msg)
task_report_path = os.path.join(output_dir, 'task-report.json')
msg = "Writing task id {i} task report to {r}".format(
r=task_report_path, i=runnable_task.task.task_id)
log.info(msg)
r.write_json(task_report_path)
return rcode, run_time
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode):
"""
:param runnable_task:
:param output_dir:
:param debug_mode:
:return:
:type runnable_task: RunnableTask
"""
def _to_p(x_):
return os.path.join(output_dir, x_)
stdout_ = _to_p('stdout')
stderr_ = _to_p('stderr')
if runnable_task.task.is_distributed is False:
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
if runnable_task.cluster is None:
log.warn("No cluster provided. Running task locally.")
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
os.chdir(runnable_task.task.output_dir)
env_json = os.path.join(output_dir, '.cluster-env.json')
IO.write_env_to_json(env_json)
# sloppy API
if isinstance(runnable_task.cluster, ClusterTemplateRender):
render = runnable_task.cluster
else:
ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()]
render = ClusterTemplateRender(ctmpls)
job_id = to_random_job_id(runnable_task.task.task_id)
log.debug("Using job id {i}".format(i=job_id))
qstdout = _to_p('cluster.stdout')
qstderr = _to_p('cluster.stderr')
qshell = _to_p('cluster.sh')
rcmd_shell = _to_p('run.sh')
# This needs to be flattened due to the new RTC layer
# Task Manifest Runner output
stdout = _to_p('stdout')
stderr = _to_p('stderr')
mstdout = _to_p('mstdout')
mstderr = _to_p('mstderr')
with open(qstdout, 'w+') as f:
f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task))
debug_str = " --debug "
exe = _resolve_exe("pbtools-runner")
_d = dict(x=exe,
t=task_manifest_path,
o=stdout,
e=stderr,
d=debug_str,
m=stdout,
n=stderr,
r=output_dir)
cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d)
with open(rcmd_shell, 'w+') as x:
x.write(cmd + "\n")
# Make +x
os.chmod(rcmd_shell, os.stat(rcmd_shell).st_mode | stat.S_IEXEC)
cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc)
log.debug(cluster_cmd)
with open(qshell, 'w') as f:
f.write("#!/bin/bash\n")
f.write("set -o errexit\n")
f.write("set -o pipefail\n")
f.write("set -o nounset\n")
f.write(cluster_cmd + "\n")
f.write("exit $?")
os.chmod(qshell, os.stat(qshell).st_mode | stat.S_IEXEC)
# host = socket.getfqdn()
host = platform.node()
# so core dumps are written to the job dir
os.chdir(output_dir)
rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell))
log.info("Cluster command return code {r} in {s:.2f} sec".format(r=rcode, s=run_time))
if rcode == 0:
err_msg = ""
warn_msg = ""
else:
p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode)
raw_stderr = _extract_last_nlines(stderr)
cluster_raw_stderr = _extract_last_nlines(qstderr)
err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr])
warn_msg = ""
msg_ = "Completed running cluster command in {t:.2f} sec. Exit code {r} (task-type {i})".format(r=rcode, t=run_time, i=runnable_task.task.task_type_id)
log.info(msg_)
with open(qstdout, 'a') as qf:
qf.write(str(cstdout) + "\n")
qf.write(msg_ + "\n")
with open(qstderr, 'a') as f:
if rcode != 0:
f.write(str(cstderr) + "\n")
f.write(msg_ + "\n")
r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg)
task_report_path = os.path.join(output_dir, 'task-report.json')
msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id)
log.info(msg)
r.write_json(task_report_path)
return rcode, err_msg, run_time
def test_help(self):
rcode, stdout, stderr, run_time = backticks(self.CMD)
self.assertEqual(rcode, 0)
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode):
"""
:param runnable_task:
:param output_dir:
:param debug_mode:
:return:
:type runnable_task: RunnableTask
"""
def _to_p(x_):
return os.path.join(output_dir, x_)
stdout_ = _to_p('stdout')
stderr_ = _to_p('stderr')
if runnable_task.task.is_distributed is False:
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
if runnable_task.cluster is None:
log.warn("No cluster provided. Running task locally.")
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
os.chdir(runnable_task.task.output_dir)
env_json = os.path.join(output_dir, '.cluster-env.json')
IO.write_env_to_json(env_json)
# sloppy API
if isinstance(runnable_task.cluster, ClusterTemplateRender):
render = runnable_task.cluster
else:
ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()]
render = ClusterTemplateRender(ctmpls)
job_id = to_random_job_id(runnable_task.task.task_id)
log.debug("Using job id {i}".format(i=job_id))
qstdout = _to_p('cluster.stdout')
qstderr = _to_p('cluster.stderr')
qshell = _to_p('cluster.sh')
rcmd_shell = _to_p('run.sh')
# This needs to be flattened due to the new RTC layer
# Task Manifest Runner output
stdout = _to_p('stdout')
stderr = _to_p('stderr')
with open(qstdout, 'w+') as f:
f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task))
debug_str = " --debug "
exe = _resolve_exe("pbtools-runner")
_d = dict(x=exe,
t=task_manifest_path,
o=stdout,
e=stderr,
d=debug_str,
m=stdout,
n=stderr,
r=output_dir)
# the quoting here is explicitly to handle spaces in paths
cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d)
# write the pbtools-runner exe command
with open(rcmd_shell, 'w+') as x:
x.write(cmd + "\n")
chmod_x(rcmd_shell)
cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc)
log.debug(cluster_cmd)
with open(qshell, 'w') as f:
f.write("#!/bin/bash\n")
f.write("set -o errexit\n")
f.write("set -o pipefail\n")
f.write("set -o nounset\n")
f.write(cluster_cmd + "\n")
f.write("exit $?")
chmod_x(qshell)
host = platform.node()
# so core dumps are written to the job dir
os.chdir(output_dir)
# Blocking call
rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell))
log.info("Cluster command return code {r} in {s:.2f} sec".format(r=rcode, s=run_time))
msg_t = "{n} Completed running cluster command in {t:.2f} sec. Exit code {r} (task-type {i})"
msg_ = msg_t.format(r=rcode, t=run_time, i=runnable_task.task.task_type_id, n=datetime.datetime.now())
log.info(msg_)
# Append the bash cluster.sh stderr and stdout call to
# the cluster.stderr and cluster.stdout
with open(qstdout, 'a') as qf:
if cstdout:
qf.write("\n".join(cstdout) + "\n")
qf.write(msg_ + "\n")
with open(qstderr, 'a') as f:
if rcode != 0:
if cstderr:
f.write(str(cstderr) + "\n")
# fundamental output error str message of this func
err_msg = ""
warn_msg = ""
if rcode != 0:
p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode)
raw_stderr = _extract_last_nlines(stderr)
cluster_raw_stderr = _extract_last_nlines(qstderr)
err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr])
warn_msg = ""
# write the result status message to stderr if task failure
# doing this here to avoid having a double message
with open(qstderr, 'a') as f:
if rcode != 0:
if cstderr:
f.write(msg_ + "\n")
r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg)
task_report_path = os.path.join(output_dir, 'task-report.json')
msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id)
log.info(msg)
r.write_json(task_report_path)
return rcode, err_msg, run_time
def test_help(self):
rcode, stdout, stderr, run_time = backticks(self.CMD)
self.assertEqual(rcode, 0)
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode):
"""
:param runnable_task:
:param output_dir:
:param debug_mode:
:return:
:type runnable_task: RunnableTask
"""
def _to_p(x_):
return os.path.join(output_dir, x_)
stdout_ = _to_p('stdout')
stderr_ = _to_p('stderr')
if runnable_task.task.is_distributed is False:
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
if runnable_task.cluster is None:
log.warn("No cluster provided. Running task locally.")
return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode)
os.chdir(runnable_task.task.output_dir)
env_json = os.path.join(output_dir, '.cluster-env.json')
IO.write_env_to_json(env_json)
# sloppy API
if isinstance(runnable_task.cluster, ClusterTemplateRender):
render = runnable_task.cluster
else:
ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()]
render = ClusterTemplateRender(ctmpls)
job_id = to_random_job_id(runnable_task.task.task_id)
log.debug("Using job id {i}".format(i=job_id))
qstdout = _to_p('cluster.stdout')
qstderr = _to_p('cluster.stderr')
qshell = _to_p('cluster.sh')
rcmd_shell = _to_p('run.sh')
# This needs to be flattened due to the new RTC layer
# Task Manifest Runner output
stdout = _to_p('stdout')
stderr = _to_p('stderr')
with open(qstdout, 'w+') as f:
f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task))
debug_str = " --debug "
exe = _resolve_exe("pbtools-runner")
_d = dict(x=exe,
t=task_manifest_path,
o=stdout,
e=stderr,
d=debug_str,
m=stdout,
n=stderr,
r=output_dir)
# the quoting here is explicitly to handle spaces in paths
cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d)
# write the pbtools-runner exe command
with open(rcmd_shell, 'w+') as x:
x.write(cmd + "\n")
chmod_x(rcmd_shell)
cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc)
log.info("Job submission command: " + cluster_cmd)
with open(qshell, 'w') as f:
f.write("#!/bin/bash\n")
f.write("set -o errexit\n")
f.write("set -o pipefail\n")
f.write("set -o nounset\n")
f.write(cluster_cmd.rstrip("\n") + " ${1+\"$@\"}\n")
f.write("exit $?")
chmod_x(qshell)
host = platform.node()
# so core dumps are written to the job dir
os.chdir(output_dir)
# print the underlying jms command if using runjmscmd
if re.search(r'/runjmscmd\b', cluster_cmd):
rcode, cstdout, cstderr, run_time = backticks("bash {q} --printcmd".format(q=qshell))
if rcode == 0:
log.info("Underlying JMS job submission command: " + "\n".join(cstdout))
# Blocking call
rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell))
log.info("Cluster command return code {r} in {s:.2f} sec".format(r=rcode, s=run_time))
msg_t = "{n} Completed running cluster command in {t:.2f} sec. Exit code {r} (task-type {i})"
msg_ = msg_t.format(r=rcode, t=run_time, i=runnable_task.task.task_type_id, n=datetime.datetime.now())
log.info(msg_)
# Append the bash cluster.sh stderr and stdout call to
# the cluster.stderr and cluster.stdout
with open(qstdout, 'a') as qf:
if cstdout:
qf.write("\n".join(cstdout) + "\n")
qf.write(msg_ + "\n")
with open(qstderr, 'a') as f:
if rcode != 0:
if cstderr:
f.write(str(cstderr) + "\n")
# fundamental output error str message of this func
err_msg = ""
warn_msg = ""
if rcode != 0:
p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode)
raw_stderr = _extract_last_nlines(stderr)
cluster_raw_stderr = _extract_last_nlines(qstderr)
err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr])
warn_msg = ""
# write the result status message to stderr if task failure
# doing this here to avoid having a double message
with open(qstderr, 'a') as f:
if rcode != 0:
if cstderr:
f.write(msg_ + "\n")
r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg)
task_report_path = os.path.join(output_dir, 'task-report.json')
msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id)
log.info(msg)
r.write_json(task_report_path)
return rcode, err_msg, run_time
