def jobs_from_array(self, array_id):
""" Creates a set of Jobs from the verbose output of the arrays.
Args:
array_id (int): Array id.
Returns:
jobs (dict): Dictionary of Job objects.
"""
cmd = "bjobs -al {}".format(array_id)
(stdout, stderr, return_code) = utils.run_shell_command(cmd)
outlines = stdout.replace("\n ", "").splitlines()
jobs = {}
joblines = []
for line in outlines:
if '---' in line:
job = LSFJob()
job.from_bjobs(joblines)
jobs[(job.jobid, job.jobindex)] = job
joblines = []
else:
joblines.append(line)
job = LSFJob()
job.from_bjobs(joblines)
jobs[(job.jobid, job.jobindex)] = job
return jobs
def sub_array_for_cmdfile(self, args):
""" Submits one or more arrays for commands in a commandfile
Args:
args (Namespace): Argparse Namespace
- split_cutoff
Returns:
array_ids (list): List of job array ids.
"""
argsd = args.__dict__
array_ids = []
for eo_file in [args.stdout, args.stderr]:
logdir = os.path.dirname(eo_file)
if logdir:
os.makedirs(logdir, exist_ok=True)
# Create commands for each bsub array needed
split_cutoff = argsd.get('split_cutoff', self._split_cutoff)
nlines = open(args.cmdfile).read().count('\n')
jid_beg = 1
jid_end = split_cutoff
if nlines < split_cutoff:
jid_end = nlines
cmd = self._get_bsub_command(argsd, jid_beg, jid_end)
(stdout, stderr, return_code) = utils.run_shell_command(cmd)
if return_code != 0:
print("ERROR: Unable to submit LSF job.")
print("bsub command: {}".format(cmd))
print("exited with code {}".format(return_code))
print("STDOUT:")
print(stdout)
print("stderr:")
print(stderr)
# NOTE: ST Jude specific LSF output:
# "Job <JOBID> is submitted to queue <QUEUE>"
array_ids.append(stdout.split('<')[1].split('>')[0])
while jid_end < nlines:
jid_beg = jid_end + 1
jid_end += split_cutoff - 1
if jid_end > nlines:
jid_end = nlines
cmd = self._get_bsub_command(argsd, jid_beg, jid_end)
(stdout, stderr, return_code) = utils.run_shell_command(cmd)
array_ids.append(stdout.split('<')[1].split('>')[0])
return array_ids
def test_command_not_found(self):
""" Tests functionality of a shell command """
(stdout, stderr, return_code) = utils.run_shell_command("blah")
self.assertIsInstance(stdout, str)
self.assertIsInstance(stderr, str)
self.assertIsInstance(return_code, int)
self.assertEqual(return_code, 127)
def test_simple_shell_command(self):
""" Tests functionality of a shell command """
(stdout, stderr, return_code) = \
utils.run_shell_command("echo foo")
self.assertIsInstance(stdout, str)
self.assertIsInstance(stderr, str)
self.assertIsInstance(return_code, int)
self.assertEqual(stdout, 'foo\n')
self.assertEqual(stderr, '')
self.assertEqual(return_code, 0)
def detect_scheduler():
""" Detect what (if any) scheduler is available.
Args:
None
Returns:
scheduler (str): The scheduler detected on the system.
"""
# Check for LSF
(stdout, stderr, return_code) = utils.run_shell_command("lsid")
if return_code == 0 and "LSF" in stdout:
return "LSF"
return "local"
def phoenix_step(directory, step, outfile=None, errfile=None, force_qc=False):
""" Run a single phoenix step.
Args:
directory (str): Directory to find scripts in.
starting_step (int): Optional step to start on.
outfile (str): Filename for stdout.
errfile (str): Filename for stderr.
force_qc (bool): Whether or not to force past QC issues
Returns: (int): Return code
"""
original_dir = os.getcwd()
assert os.path.isdir(directory)
directory = os.path.realpath(directory)
fout = sys.stdout
if outfile:
fout = open(outfile, 'w')
ferr = sys.stderr
if errfile:
ferr = open(errfile, 'w')
steps = utils.get_phoenix_steps(directory)
assert steps
step_name = steps[step]
print("[STEP {}] Running the following substeps:".format(step), file=fout)
all_phoenix_scripts = utils.get_phoenix_scripts(directory)
base_regex = "%02i" % (step) + "_?_*_*.sh"
phoenix_scripts = glob.glob(os.path.join(directory, base_regex))
substeps = []
# Get a list of all substeps to be processed (in order)
for phoenix_script in sorted(phoenix_scripts):
assert phoenix_script in all_phoenix_scripts
substep_script = os.path.basename(phoenix_script)
pieces = substep_script.split('.sh')[0].split('_')
substeps.append(
(step, int(pieces[1]), pieces[3], pieces[2], phoenix_script))
if len(substeps) > 1:
assert substeps[-1][0] > substeps[-2][0] \
or substeps[-1][1] > substeps[-2][1]
# Print out the substeps to be processed
for substep in substeps:
print("[STEP {}.{}] {}.{}".format(substep[0], substep[1], substep[2],
substep[3]),
file=fout)
# Process the substeps
for substep in substeps:
stat = "[STEP {}.{}]".format(substep[0], substep[1])
step_name = substep[2]
substep_name = substep[3]
if substep_name == "qc" and force_qc:
print("{} SKIP qc".format(stat))
continue
phoenix_script = substep[4]
print("", file=fout)
print("{} STEP {}.{}".format(stat, step_name, substep_name), file=fout)
print("{} Date: ".format(stat), datetime.datetime.now(), file=fout)
print("{} Running script {}".format(stat, phoenix_script), file=fout)
os.chdir(directory)
(stdout, stderr, return_code) = \
utils.run_shell_command(phoenix_script, stdout=fout, stderr=ferr)
os.chdir(original_dir)
if return_code != 0:
print("{} Substep FAILURE".format(stat), file=fout)
return return_code
print("{} Done running {}".format(stat, phoenix_script), file=fout)
print("{} Date: ".format(stat), datetime.datetime.now(), file=fout)
print("{} Substep successful!".format(stat), file=fout)
print("", file=fout)
print("[STEP {}] Step successful!".format(step), file=fout)
if outfile:
fout.close()
if errfile:
ferr.close()
return 0