def test_write_pipeline_preset_json(self): ofn = tempfile.NamedTemporaryFile(suffix=".json").name CMD = "pbsmrtpipe show-template-details pbsmrtpipe.pipelines.dev_diagnostic_stress -j {f}".format(f=ofn) rcode, stdout, stderr, run_time = backticks(CMD) self.assertEqual(rcode, 0) with open(ofn) as f: d = json.load(f) self.assertEqual(len(d['taskOptions']), 2)
def test_write_workflow_preset_json(self): from pbsmrtpipe.pb_io import REGISTERED_WORKFLOW_OPTIONS ofn = tempfile.NamedTemporaryFile(suffix=".json").name CMD = "pbsmrtpipe show-workflow-options -j {f}".format(f=ofn) rcode, stdout, stderr, run_time = backticks(CMD) self.assertEqual(rcode, 0) with open(ofn) as f: d = json.load(f) self.assertEqual(len(d['options']), len(REGISTERED_WORKFLOW_OPTIONS))
def test_write_pipeline_preset_json(self): ofn = tempfile.NamedTemporaryFile(suffix=".json").name pipeline_id = "pbsmrtpipe.pipelines.dev_diagnostic_stress" cmd = "pbsmrtpipe show-template-details {i} -j {f}".format(f=ofn, i=pipeline_id) rcode, stdout, stderr, run_time = backticks(cmd) fail_msg = "Failed (exit code {e}) to show template details cmd: {c} {m} ".format(i=pipeline_id, e=rcode, c=cmd, m=str(stderr)) self.assertEqual(rcode, 0, fail_msg) with open(ofn) as f: d = json.load(f) self.assertEqual(len(d['taskOptions']), 9)
def _run_testkit_cfg(testkit_cfg, debug=False, misc_opts=Constants.MISC_OPTS): os.chdir(os.path.dirname(testkit_cfg)) cmd = "{e} --debug {m} {c}".format(c=testkit_cfg, e=Constants.EXE, m=misc_opts) rcode, stdout, stderr, run_time = backticks(cmd) if debug: log.debug(" ".join([str(i) for i in [cmd, rcode, stdout, stderr]])) # Returning the butler cfg is a bit odd, but this is necessary for the # parallelism to work consistently with the serial version return testkit_cfg, rcode, stdout, stderr, run_time
def _run_testkit_cfg(testkit_cfg, debug=False, misc_opts=""): os.chdir(os.path.dirname(testkit_cfg)) cmd = "{e} --debug {m} {c}".format(c=testkit_cfg, e=_EXE, m=misc_opts) rcode, stdout, stderr, run_time = backticks(cmd) if debug: log.debug(" ".join([str(i) for i in [cmd, rcode, stdout, stderr]])) # Returning the butler cfg is a bit odd, but this is necessary for the # parallelism to work consistently with the serial version return testkit_cfg, rcode, stdout, stderr, run_time
def test_backticks(self): exe = 'python --version' rcode, out, err, run_time = backticks(exe) self.assertEqual(rcode, 0) self.assertTrue(out[0].startswith('Python ')) self.assertEqual(err, "")
def _run_testkit_cfg(testkit_cfg, debug=False, misc_opts=Constants.MISC_OPTS, sleep_time=Constants.SLEEP_TIME): time.sleep(random.randint(1, sleep_time * 2)) os.chdir(os.path.dirname(testkit_cfg)) cmd = "{e} --debug --sleep {t} {m} {c}".format(c=testkit_cfg, e=Constants.EXE, m=misc_opts, t=sleep_time) rcode, stdout, stderr, run_time = backticks(cmd) if debug: log.debug(" ".join([str(i) for i in [cmd, rcode, stdout, stderr]])) # Returning the butler cfg is a bit odd, but this is necessary for the # parallelism to work consistently with the serial version return testkit_cfg, rcode, stdout, stderr, run_time
def _dot_to_image(image_type, dot_file, image_file): assert image_type.lower() in _SUPPORTED_IMAGE_TYPES if not os.path.exists(dot_file): raise IOError("Unable to find {f}".format(f=dot_file)) if which(DOT_EXE) is None: raise RequiredExeNotFoundError("Unable to find required external exe '{x}'".format(x=DOT_EXE)) cmd_str = "{e} -T{t} {i} -o {o}" d = dict(e=DOT_EXE, t=image_type, i=dot_file, o=image_file) cmd = cmd_str.format(**d) rcode, stdout, stderr, run_time = backticks(cmd) state = True if rcode == 0 else False return state
def _dot_to_image(image_type, dot_file, image_file): assert image_type.lower() in _SUPPORTED_IMAGE_TYPES if not os.path.exists(dot_file): raise IOError("Unable to find {f}".format(f=dot_file)) if which(DOT_EXE) is None: raise RequiredExeNotFoundError( "Unable to find required external exe '{x}'".format(x=DOT_EXE)) cmd_str = "{e} -T{t} {i} -o {o}" d = dict(e=DOT_EXE, t=image_type, i=dot_file, o=image_file) cmd = cmd_str.format(**d) rcode, stdout, stderr, run_time = backticks(cmd) state = True if rcode == 0 else False return state
def _align_to_reference(sequence, reference, job_dir): alignment_output = tempfile.NamedTemporaryFile( suffix="_consensus.out").name cmd = " ".join([ "blasr", sequence, reference, "--minMatch", "15", "--maxMatch", "20", "--fastMaxInterval", "-m", "5", "--fastSDP", "--aggressiveIntervalCut", "--advanceExactMatches", "10", "--out", alignment_output]) errCode, output, errorMessage, run_time = backticks(cmd) log.debug(cmd) if errCode is 0: log.info(output) elif errCode is 1: log.error(errorMessage) return alignment_output
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode): """ :param runnable_task: :param output_dir: :param debug_mode: :return: :type runnable_task: RunnableTask """ def _to_p(x_): return os.path.join(output_dir, x_) stdout_ = _to_p('stdout') stderr_ = _to_p('stderr') if runnable_task.task.is_distributed is False: return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) if runnable_task.cluster is None: log.warn("No cluster provided. Running task locally.") return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) os.chdir(runnable_task.task.output_dir) env_json = os.path.join(output_dir, '.cluster-env.json') IO.write_env_to_json(env_json) # sloppy API if isinstance(runnable_task.cluster, ClusterTemplateRender): render = runnable_task.cluster else: ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()] render = ClusterTemplateRender(ctmpls) job_id = to_random_job_id(runnable_task.task.task_id) log.debug("Using job id {i}".format(i=job_id)) qstdout = _to_p('cluster.stdout') qstderr = _to_p('cluster.stderr') qshell = _to_p('cluster.sh') rcmd_shell = _to_p('run.sh') # This needs to be flattened due to the new RTC layer # Task Manifest Runner output stdout = _to_p('stdout') stderr = _to_p('stderr') mstdout = _to_p('mstdout') mstderr = _to_p('mstderr') with open(qstdout, 'w+') as f: f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task)) debug_str = " --debug " exe = _resolve_exe("pbtools-runner") _d = dict(x=exe, t=task_manifest_path, o=stdout, e=stderr, d=debug_str, m=stdout, n=stderr, r=output_dir) cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d) with open(rcmd_shell, 'w+') as x: x.write(cmd + "\n") # Make +x os.chmod(rcmd_shell, os.stat(rcmd_shell).st_mode | stat.S_IEXEC) cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc) log.debug(cluster_cmd) with open(qshell, 'w') as f: f.write("#!/bin/bash\n") f.write(cluster_cmd + "\n") os.chmod(qshell, os.stat(qshell).st_mode | stat.S_IEXEC) # host = socket.getfqdn() host = platform.node() # so core dumps are written to the job dir os.chdir(output_dir) rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell)) if rcode == 0: err_msg = "" warn_msg = "" else: p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode) raw_stderr = _extract_last_nlines(stderr) cluster_raw_stderr = _extract_last_nlines(cstderr) err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr]) warn_msg = "" msg_ = "Completed running cluster command in {t:.2f} sec. Exit code {r}".format(r=rcode, t=run_time) log.info(msg_) with open(qstdout, 'a') as qf: qf.write(str(cstdout) + "\n") qf.write(msg_ + "\n") with open(qstderr, 'a') as f: if rcode != 0: f.write(str(cstderr) + "\n") f.write(msg_ + "\n") r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg) task_report_path = os.path.join(output_dir, 'task-report.json') msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id) log.info(msg) r.write_json(task_report_path) return rcode, run_time
def was_backticks_successful(cmd): rcode, _, _, _ = backticks(cmd, ) return rcode == 0
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode): """ :param runnable_task: :param output_dir: :param debug_mode: :return: :type runnable_task: RunnableTask """ def _to_p(x_): return os.path.join(output_dir, x_) stdout_ = _to_p('stdout') stderr_ = _to_p('stderr') if runnable_task.task.is_distributed is False: return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) if runnable_task.cluster is None: log.warn("No cluster provided. Running task locally.") return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) env_json = os.path.join(output_dir, 'env.json') IO.write_env_to_json(env_json) # sloppy API if isinstance(runnable_task.cluster, ClusterTemplateRender): render = runnable_task.cluster else: ctmpls = [ ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems() ] render = ClusterTemplateRender(ctmpls) job_id = to_random_job_id(runnable_task.task.task_id) log.debug("Using job id {i}".format(i=job_id)) qstdout = _to_p('cluster.stdout') qstderr = _to_p('cluster.stderr') qshell = _to_p('cluster.sh') rcmd_shell = _to_p('run.sh') # Task Manifest Runner output stdout = _to_p('stdout') stderr = _to_p('stderr') mstdout = _to_p('mstdout') mstderr = _to_p('mstderr') with open(qstdout, 'w+') as f: f.write("Creating cluster stdout for Job {i} {r}\n".format( i=job_id, r=runnable_task)) debug_str = " --debug " _d = dict(t=task_manifest_path, o=stdout, e=stderr, d=debug_str, m=mstdout, n=mstderr) cmd = "pbtools-runner run {d} --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format( **_d) with open(rcmd_shell, 'w+') as x: x.write(cmd + "\n") # Make +x os.chmod(rcmd_shell, os.stat(rcmd_shell).st_mode | stat.S_IEXEC) cluster_cmd = render.render('interactive', rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc) log.debug(cluster_cmd) with open(qshell, 'w') as f: f.write("#!/bin/bash\n") f.write(cluster_cmd + "\n") os.chmod(qshell, os.stat(qshell).st_mode | stat.S_IEXEC) # host = socket.getfqdn() host = platform.node() # so core dumps are written to the job dir os.chdir(output_dir) rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell)) if rcode == 0: err_msg = "" warn_msg = "" else: # not sure how to scrape this from the stderr/stdout err_msg = "task {i} failed".format(i=runnable_task.task.task_id) warn_msg = "" msg_ = "Completed running cluster command in {t:.2f} sec. Exit code {r}".format( r=rcode, t=run_time) log.info(msg_) with open(qstdout, 'a') as qf: qf.write(str(cstdout) + "\n") qf.write(msg_ + "\n") with open(qstderr, 'a') as f: if rcode != 0: f.write(str(cstderr) + "\n") f.write(msg_ + "\n") r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg) task_report_path = os.path.join(output_dir, 'task-report.json') msg = "Writing task id {i} task report to {r}".format( r=task_report_path, i=runnable_task.task.task_id) log.info(msg) r.write_json(task_report_path) return rcode, run_time
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode): """ :param runnable_task: :param output_dir: :param debug_mode: :return: :type runnable_task: RunnableTask """ def _to_p(x_): return os.path.join(output_dir, x_) stdout_ = _to_p('stdout') stderr_ = _to_p('stderr') if runnable_task.task.is_distributed is False: return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) if runnable_task.cluster is None: log.warn("No cluster provided. Running task locally.") return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) os.chdir(runnable_task.task.output_dir) env_json = os.path.join(output_dir, '.cluster-env.json') IO.write_env_to_json(env_json) # sloppy API if isinstance(runnable_task.cluster, ClusterTemplateRender): render = runnable_task.cluster else: ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()] render = ClusterTemplateRender(ctmpls) job_id = to_random_job_id(runnable_task.task.task_id) log.debug("Using job id {i}".format(i=job_id)) qstdout = _to_p('cluster.stdout') qstderr = _to_p('cluster.stderr') qshell = _to_p('cluster.sh') rcmd_shell = _to_p('run.sh') # This needs to be flattened due to the new RTC layer # Task Manifest Runner output stdout = _to_p('stdout') stderr = _to_p('stderr') mstdout = _to_p('mstdout') mstderr = _to_p('mstderr') with open(qstdout, 'w+') as f: f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task)) debug_str = " --debug " exe = _resolve_exe("pbtools-runner") _d = dict(x=exe, t=task_manifest_path, o=stdout, e=stderr, d=debug_str, m=stdout, n=stderr, r=output_dir) cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d) with open(rcmd_shell, 'w+') as x: x.write(cmd + "\n") # Make +x os.chmod(rcmd_shell, os.stat(rcmd_shell).st_mode | stat.S_IEXEC) cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc) log.debug(cluster_cmd) with open(qshell, 'w') as f: f.write("#!/bin/bash\n") f.write("set -o errexit\n") f.write("set -o pipefail\n") f.write("set -o nounset\n") f.write(cluster_cmd + "\n") f.write("exit $?") os.chmod(qshell, os.stat(qshell).st_mode | stat.S_IEXEC) # host = socket.getfqdn() host = platform.node() # so core dumps are written to the job dir os.chdir(output_dir) rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell)) log.info("Cluster command return code {r} in {s:.2f} sec".format(r=rcode, s=run_time)) if rcode == 0: err_msg = "" warn_msg = "" else: p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode) raw_stderr = _extract_last_nlines(stderr) cluster_raw_stderr = _extract_last_nlines(qstderr) err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr]) warn_msg = "" msg_ = "Completed running cluster command in {t:.2f} sec. Exit code {r} (task-type {i})".format(r=rcode, t=run_time, i=runnable_task.task.task_type_id) log.info(msg_) with open(qstdout, 'a') as qf: qf.write(str(cstdout) + "\n") qf.write(msg_ + "\n") with open(qstderr, 'a') as f: if rcode != 0: f.write(str(cstderr) + "\n") f.write(msg_ + "\n") r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg) task_report_path = os.path.join(output_dir, 'task-report.json') msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id) log.info(msg) r.write_json(task_report_path) return rcode, err_msg, run_time
def test_help(self): rcode, stdout, stderr, run_time = backticks(self.CMD) self.assertEqual(rcode, 0)
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode): """ :param runnable_task: :param output_dir: :param debug_mode: :return: :type runnable_task: RunnableTask """ def _to_p(x_): return os.path.join(output_dir, x_) stdout_ = _to_p('stdout') stderr_ = _to_p('stderr') if runnable_task.task.is_distributed is False: return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) if runnable_task.cluster is None: log.warn("No cluster provided. Running task locally.") return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) os.chdir(runnable_task.task.output_dir) env_json = os.path.join(output_dir, '.cluster-env.json') IO.write_env_to_json(env_json) # sloppy API if isinstance(runnable_task.cluster, ClusterTemplateRender): render = runnable_task.cluster else: ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()] render = ClusterTemplateRender(ctmpls) job_id = to_random_job_id(runnable_task.task.task_id) log.debug("Using job id {i}".format(i=job_id)) qstdout = _to_p('cluster.stdout') qstderr = _to_p('cluster.stderr') qshell = _to_p('cluster.sh') rcmd_shell = _to_p('run.sh') # This needs to be flattened due to the new RTC layer # Task Manifest Runner output stdout = _to_p('stdout') stderr = _to_p('stderr') with open(qstdout, 'w+') as f: f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task)) debug_str = " --debug " exe = _resolve_exe("pbtools-runner") _d = dict(x=exe, t=task_manifest_path, o=stdout, e=stderr, d=debug_str, m=stdout, n=stderr, r=output_dir) # the quoting here is explicitly to handle spaces in paths cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d) # write the pbtools-runner exe command with open(rcmd_shell, 'w+') as x: x.write(cmd + "\n") chmod_x(rcmd_shell) cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc) log.debug(cluster_cmd) with open(qshell, 'w') as f: f.write("#!/bin/bash\n") f.write("set -o errexit\n") f.write("set -o pipefail\n") f.write("set -o nounset\n") f.write(cluster_cmd + "\n") f.write("exit $?") chmod_x(qshell) host = platform.node() # so core dumps are written to the job dir os.chdir(output_dir) # Blocking call rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell)) log.info("Cluster command return code {r} in {s:.2f} sec".format(r=rcode, s=run_time)) msg_t = "{n} Completed running cluster command in {t:.2f} sec. Exit code {r} (task-type {i})" msg_ = msg_t.format(r=rcode, t=run_time, i=runnable_task.task.task_type_id, n=datetime.datetime.now()) log.info(msg_) # Append the bash cluster.sh stderr and stdout call to # the cluster.stderr and cluster.stdout with open(qstdout, 'a') as qf: if cstdout: qf.write("\n".join(cstdout) + "\n") qf.write(msg_ + "\n") with open(qstderr, 'a') as f: if rcode != 0: if cstderr: f.write(str(cstderr) + "\n") # fundamental output error str message of this func err_msg = "" warn_msg = "" if rcode != 0: p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode) raw_stderr = _extract_last_nlines(stderr) cluster_raw_stderr = _extract_last_nlines(qstderr) err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr]) warn_msg = "" # write the result status message to stderr if task failure # doing this here to avoid having a double message with open(qstderr, 'a') as f: if rcode != 0: if cstderr: f.write(msg_ + "\n") r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg) task_report_path = os.path.join(output_dir, 'task-report.json') msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id) log.info(msg) r.write_json(task_report_path) return rcode, err_msg, run_time
def run_task_on_cluster(runnable_task, task_manifest_path, output_dir, debug_mode): """ :param runnable_task: :param output_dir: :param debug_mode: :return: :type runnable_task: RunnableTask """ def _to_p(x_): return os.path.join(output_dir, x_) stdout_ = _to_p('stdout') stderr_ = _to_p('stderr') if runnable_task.task.is_distributed is False: return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) if runnable_task.cluster is None: log.warn("No cluster provided. Running task locally.") return run_task(runnable_task, output_dir, stdout_, stderr_, debug_mode) os.chdir(runnable_task.task.output_dir) env_json = os.path.join(output_dir, '.cluster-env.json') IO.write_env_to_json(env_json) # sloppy API if isinstance(runnable_task.cluster, ClusterTemplateRender): render = runnable_task.cluster else: ctmpls = [ClusterTemplate(name, tmpl) for name, tmpl in runnable_task.cluster.iteritems()] render = ClusterTemplateRender(ctmpls) job_id = to_random_job_id(runnable_task.task.task_id) log.debug("Using job id {i}".format(i=job_id)) qstdout = _to_p('cluster.stdout') qstderr = _to_p('cluster.stderr') qshell = _to_p('cluster.sh') rcmd_shell = _to_p('run.sh') # This needs to be flattened due to the new RTC layer # Task Manifest Runner output stdout = _to_p('stdout') stderr = _to_p('stderr') with open(qstdout, 'w+') as f: f.write("Creating cluster stdout for Job {i} {r}\n".format(i=job_id, r=runnable_task)) debug_str = " --debug " exe = _resolve_exe("pbtools-runner") _d = dict(x=exe, t=task_manifest_path, o=stdout, e=stderr, d=debug_str, m=stdout, n=stderr, r=output_dir) # the quoting here is explicitly to handle spaces in paths cmd = "{x} run {d} --output-dir=\"{r}\" --task-stderr=\"{e}\" --task-stdout=\"{o}\" \"{t}\" > \"{m}\" 2> \"{n}\"".format(**_d) # write the pbtools-runner exe command with open(rcmd_shell, 'w+') as x: x.write(cmd + "\n") chmod_x(rcmd_shell) cluster_cmd = render.render(ClusterConstants.START, rcmd_shell, job_id, qstdout, qstderr, runnable_task.task.nproc) log.info("Job submission command: " + cluster_cmd) with open(qshell, 'w') as f: f.write("#!/bin/bash\n") f.write("set -o errexit\n") f.write("set -o pipefail\n") f.write("set -o nounset\n") f.write(cluster_cmd.rstrip("\n") + " ${1+\"$@\"}\n") f.write("exit $?") chmod_x(qshell) host = platform.node() # so core dumps are written to the job dir os.chdir(output_dir) # print the underlying jms command if using runjmscmd if re.search(r'/runjmscmd\b', cluster_cmd): rcode, cstdout, cstderr, run_time = backticks("bash {q} --printcmd".format(q=qshell)) if rcode == 0: log.info("Underlying JMS job submission command: " + "\n".join(cstdout)) # Blocking call rcode, cstdout, cstderr, run_time = backticks("bash {q}".format(q=qshell)) log.info("Cluster command return code {r} in {s:.2f} sec".format(r=rcode, s=run_time)) msg_t = "{n} Completed running cluster command in {t:.2f} sec. Exit code {r} (task-type {i})" msg_ = msg_t.format(r=rcode, t=run_time, i=runnable_task.task.task_type_id, n=datetime.datetime.now()) log.info(msg_) # Append the bash cluster.sh stderr and stdout call to # the cluster.stderr and cluster.stdout with open(qstdout, 'a') as qf: if cstdout: qf.write("\n".join(cstdout) + "\n") qf.write(msg_ + "\n") with open(qstderr, 'a') as f: if rcode != 0: if cstderr: f.write(str(cstderr) + "\n") # fundamental output error str message of this func err_msg = "" warn_msg = "" if rcode != 0: p_err_msg = "task {i} failed (exit-code {x}) after {r:.2f} sec".format(i=runnable_task.task.task_id, r=run_time, x=rcode) raw_stderr = _extract_last_nlines(stderr) cluster_raw_stderr = _extract_last_nlines(qstderr) err_msg = "\n".join([p_err_msg, raw_stderr, cluster_raw_stderr]) warn_msg = "" # write the result status message to stderr if task failure # doing this here to avoid having a double message with open(qstderr, 'a') as f: if rcode != 0: if cstderr: f.write(msg_ + "\n") r = to_task_report(host, runnable_task.task.task_id, run_time, rcode, err_msg, warn_msg) task_report_path = os.path.join(output_dir, 'task-report.json') msg = "Writing task id {i} task report to {r}".format(r=task_report_path, i=runnable_task.task.task_id) log.info(msg) r.write_json(task_report_path) return rcode, err_msg, run_time