def test_job_arg_name_files(tmpdir, fancyname):
# Test to check that when a named argument is added to a Job, and the Job
# is built with fancyname=True, the Job submit file and the
# error/log/output files for the argument start with the same index.
# E.g. job_(date)_01.submit, job_(date)_01.error, etc.
# Regression test for issue #47
submit_dir = str(tmpdir.mkdir('submit'))
job = Job('testjob', example_script, submit=submit_dir)
job.add_arg('arg', name='argname')
dagman = Dagman('exampledagman', submit=submit_dir)
dagman.add_job(job)
dagman.build(fancyname=fancyname)
with open(dagman.submit_file, 'r') as dagman_submit_file:
dagman_submit_lines = dagman_submit_file.readlines()
# Get root of the dagman submit file (submit file basename w/o .submit)
submit_file_line = dagman_submit_lines[0]
submit_file_basename = submit_file_line.split(os.sep)[-1].rstrip()
submit_file_root = os.path.splitext(submit_file_basename)[0]
# Get job_name variable (used to built error/log/output file basenames)
jobname_line = dagman_submit_lines[2]
jobname = jobname_line.split('"')[-2]
other_file_root = '_'.join(jobname.split('_')[:-1])
assert submit_file_root == other_file_root
def test_iter_job_args_fail(tmpdir):
submit_dir = str(tmpdir.mkdir('submit'))
# Check _iter_job_args raises a ValueError if input Job is not built
job = Job('testjob', example_script, submit=submit_dir)
with pytest.raises(ValueError) as excinfo:
i = _iter_job_args(job)
node_name, arg = next(i)
error = ('Job {} must be built before adding it to a '
'Dagman'.format(job.name))
assert error == str(excinfo.value)
# Check _iter_job_args raises a StopIteration exception on a Job w/o args
job.build()
with pytest.raises(StopIteration):
i = _iter_job_args(job)
node_name, arg = next(i)
# Check _iter_job_args raises a TypeError when input is not a Job
with pytest.raises(TypeError) as excinfo:
not_job = 'thisisastring'
i = _iter_job_args(not_job)
node_name, arg = next(i)
error = 'Expecting a Job object, got {}'.format(type(not_job))
assert error == str(excinfo.value)
def test_build_executeable_not_found_fail(tmpdir):
submit_dir = str(tmpdir.mkdir('submit'))
with pytest.raises(IOError) as excinfo:
ex = '/path/to/executable'
job = Job('jobname', ex, submit=submit_dir)
job.build(makedirs=False)
error = 'The executable {} does not exist'.format(ex)
assert error == str(excinfo.value)
def test_add_job_dag_parameter_equality(tmpdir):
submit_dir = str(tmpdir.join('submit'))
dag = Dagman('dagman', submit=submit_dir)
job_1 = Job('job_1', example_script, dag=dag)
job_2 = Job('job_2', example_script)
dag.add_job(job_2)
assert dag.nodes == [job_1, job_2]
def test_init_arguments_type_fail():
with pytest.raises(TypeError) as excinfo:
job_with_arg = Job(name='jobname',
executable=example_script,
arguments=50)
job_with_arg.build()
error = 'arguments must be a string or an iterable'
assert error == str(excinfo.value)
def test_job_submit_env_variable_dir(tmpdir, monkeypatch):
# Use monkeypatch fixture to set pycondor environment variable
dir_path = str(tmpdir.mkdir('submit'))
monkeypatch.setenv('PYCONDOR_SUBMIT_DIR', dir_path)
job = Job('jobname', example_script)
job.build()
tmpdir_path = os.path.join(str(tmpdir), 'submit')
job_path = os.path.dirname(getattr(job, 'submit_file'))
assert tmpdir_path == job_path
def test_job_env_variable_dir(tmpdir, monkeypatch, env_var):
submit_dir = str(tmpdir)
# Use monkeypatch fixture to set pycondor environment variable
dir_path = str(tmpdir.mkdir(env_var))
monkeypatch.setenv('PYCONDOR_{}_DIR'.format(env_var.upper()), dir_path)
job = Job('jobname', example_script, submit=submit_dir)
job.build()
tmpdir_path = os.path.join(str(tmpdir), env_var)
job_path = os.path.dirname(getattr(job, '{}_file'.format(env_var)))
assert tmpdir_path == job_path
def test_job_subdag_build(tmpdir):
submit_dir = str(tmpdir.join('submit'))
extra_lines = ['first extra line', 'second extra line']
job = Job('job',
example_script,
submit=submit_dir,
extra_lines=extra_lines)
job.build()
with open(job.submit_file, 'r') as f:
assert set(extra_lines) <= set(line.rstrip('\n') for line in f)
def test_repr():
default_job = Job('jobname', example_script)
job_repr = repr(default_job)
expected_repr = ('Job(name=jobname, executable=example_script.py, '
'submit={})'.format(os.getcwd()))
assert job_repr == expected_repr
job_non_default = Job('jobname', example_script, queue=2)
job_repr = repr(job_non_default)
expected_repr = ('Job(name=jobname, executable=example_script.py, '
'queue=2, submit={})'.format(os.getcwd()))
assert job_repr == expected_repr
def test_repr():
default_job = Job('jobname', example_script)
job_repr = repr(default_job)
expected_repr = ('Job(name=jobname, executable=example_script.py, '
'getenv=True, notification=never, submit={}, '
'universe=vanilla)'.format(os.getcwd()))
assert job_repr == expected_repr
job_non_default = Job('jobname', example_script, queue=2)
job_repr = repr(job_non_default)
expected_repr = ('Job(name=jobname, executable=example_script.py, '
'getenv=True, notification=never, queue=2, submit={}, '
'universe=vanilla)'.format(os.getcwd()))
assert job_repr == expected_repr
def test_dagman_has_bad_node_names(tmpdir):
submit_dir = str(tmpdir.mkdir('submit'))
# Test all combinations
jobs_names = ['testjob', 'testjob.', 'testjob', 'testjob+']
arg_names = ['argname', 'argname', 'argname+', 'argname.']
has_bad_node_names = [False, True, True, True]
for job_name, arg_name, bad_node_names in zip(jobs_names, arg_names,
has_bad_node_names):
job = Job(job_name, example_script, submit=submit_dir)
job.add_arg('arg', name=arg_name)
dagman = Dagman('testdagman', submit=submit_dir)
dagman.add_job(job)
dagman.build()
assert dagman._has_bad_node_names == bad_node_names
def test_queue_written_to_submit_file(tmpdir):
# Test to check that the queue parameter is properly written
# to submit file when Job is created. See issue #38.
submit_dir = str(tmpdir.mkdir('submit'))
# Build Job object with queue=5
job = Job('jobname', example_script, submit=submit_dir, queue=5)
job.build(fancyname=False)
# Read the built submit file and check that the 'queue 5' is
# contained in the file.
with open(job.submit_file, 'r') as f:
lines = f.readlines()
assert 'queue 5' in lines
def test_submit_job_parents_raises(tmpdir, monkeypatch_condor_submit):
# Test submitting a Job with parents (not in a Dagman) raises an error
submit = str(tmpdir)
job = Job('jobname', example_script, submit=submit)
parent_job = Job('parent_jobname', example_script, submit=submit)
job.add_parent(parent_job)
job.build()
with pytest.raises(ValueError) as excinfo:
job.submit_job()
error = ('Attempting to submit a Job with parents. '
'Interjob relationships requires Dagman.')
assert error == str(excinfo.value)
def test_dagman_add_node_ignores_duplicates(tmpdir, dagman):
submit_dir = str(tmpdir.join('submit'))
job = Job('job', example_script, submit=submit_dir)
dagman.add_job(job)
dagman.add_job(job)
assert dagman.nodes == [job]
def test_get_job_arg_lines_not_built_raises():
job = Job('testjob', example_script)
with pytest.raises(ValueError) as excinfo:
Dagman('dag_name')._get_job_arg_lines(job, fancyname=True)
error = ('Job {} must be built before adding it to a '
'Dagman'.format(job.name))
assert error == str(excinfo.value)
def test_job_dag_parameter(tmpdir):
# Test that a Job is added to a Dagman when dag parameter given
submit_dir = str(tmpdir.join('submit'))
dag = Dagman('dagman', submit=submit_dir)
job = Job('job', example_script, dag=dag)
assert job in dag
def test_add_args():
# Test that add_args is equivalent to multiple add_arg
job_1 = Job('job1', example_script)
for i in range(10):
job_1.add_arg('file_{}.hdf'.format(i))
job_2 = Job('job2', example_script)
job_2.add_args(['file_{}.hdf'.format(i) for i in range(10)])
assert job_1.args == job_2.args
def test_iter_job_args(tmpdir):
# Check node names yielded by _iter_job_args
submit_dir = str(tmpdir.mkdir('submit'))
job = Job('testjob', example_script, submit=submit_dir)
job.add_arg('argument1', name='arg1')
job.add_arg('argument2')
job.build()
for idx, (node_name, jobarg) in enumerate(_iter_job_args(job)):
if jobarg.name is not None:
assert node_name == '{}_{}'.format(job.submit_name, jobarg.name)
else:
assert node_name == '{}_arg_{}'.format(job.submit_name, idx)
def test_dagman_env_variable_dir(tmpdir, monkeypatch):
# Set pycondor environment variable
submit_dir = str(tmpdir.mkdir('submit'))
monkeypatch.setenv('PYCONDOR_SUBMIT_DIR', submit_dir)
dagman = Dagman('testdagman')
job = Job('jobname', example_script)
dagman.add_job(job)
dagman.build()
submit_path = os.path.dirname(dagman.submit_file)
assert submit_dir == submit_path
def test_job_dag_submit_file_same(tmpdir, dagman):
# Test to check that the submit file for a Job with no arguments is the
# same whether built from a Dagman or not. See issue #38.
submit_dir = str(tmpdir.mkdir('submit'))
# Build Job object that will be built outside of a Dagman
job_outside_dag = Job('test_job',
example_script,
submit=submit_dir,
queue=5)
job_outside_dag.build(fancyname=False)
# Build Job object that will be built inside of a Dagman
job_inside_dag = Job('test_job',
example_script,
submit=submit_dir,
queue=5)
dagman.add_job(job_inside_dag)
dagman.build(fancyname=False)
# Check that the contents of the two Job submit files are the same
assert filecmp.cmp(job_outside_dag.submit_file,
job_inside_dag.submit_file,
shallow=False)
def test_init_retry():
# Test that global retry applies to add_arg without a retry specified and
# not when add_arg has a retry specified
job = Job(name='jobname', executable=example_script, retry=7)
job.add_arg('arg1')
job.add_arg('arg2', retry=3)
assert len(job.args) == 2
assert job.args[0].retry == 7
assert job.args[1].retry == 3
def test_job_args_and_queue_raises(tmpdir):
submit_dir = str(tmpdir.join('submit'))
with pytest.raises(NotImplementedError) as excinfo:
job = Job('job', example_script, submit=submit_dir, queue=2)
job.add_args(str(i) for i in range(10))
job.build()
error = ('At this time multiple arguments and queue values '
'are only supported through Dagman')
assert error == str(excinfo.value)
def ppplots(self):
"""
Set up job to create PP plots.
"""
from pycondor import Job
# get executable
jobexec = shutil.which("cwinpy_pe_generate_pp_plots")
extra_lines = []
if self.accountgroup is not None:
extra_lines.append("accounting_group = {}".format(
self.accountgroup))
if self.accountuser is not None:
extra_lines.append("accounting_group_user = {}".format(
self.accountuser))
# create cwinpy_pe Job
job = Job(
"cwinpy_pe_pp_plots",
jobexec,
error=self.runner.dag.inputs.pe_log_directory,
log=self.runner.dag.inputs.pe_log_directory,
output=self.runner.dag.inputs.pe_log_directory,
submit=self.runner.dag.inputs.submit_directory,
universe="vanilla",
request_memory=self.runner.dag.inputs.request_memory,
getenv=self.getenv,
queue=1,
requirements=self.runner.dag.inputs.requirements,
retry=self.runner.dag.inputs.retry,
extra_lines=extra_lines,
dag=self.runner.dag.pycondor_dag,
)
jobargs = "--path '{}' ".format(
os.path.join(self.basedir, "results", "*", "*"))
jobargs += "--output {} ".format(
os.path.join(self.basedir, "ppplot.png"))
if self.outputsnr:
jobargs += "--snrs "
job.add_arg(jobargs)
job.add_parents(self.runner.dag.pycondor_dag.nodes[:-1]
) # exclude cwinpy_pe_pp_plots job itself
self.runner.dag.build()
# Get script and options
generator_script = "/home/fasig/scalable_radio_array/noise_writer.sh"
processor_script = "/home/fasig/scalable_radio_array/envelope_processor.sh"
# Declare the error, output, log, and submit directories for Condor Job
error = '/data/user/fasig/pycondor'
output = '/data/user/fasig/pycondor'
log = '/data/user/fasig/pycondor'
submit = '/data/user/fasig/pycondor'
# Setting up PyCondor Jobs
generator_job = Job("make_noise_" + basename,
generator_script,
error=error,
output=output,
log=log,
submit=submit,
verbose=2)
processor_job = Job("process_noise_" + basename,
processor_script,
error=error,
output=output,
log=log,
submit=submit,
verbose=2)
output_suffixes = [
str(i).zfill(file_zero_padding) for i in range(max_file_index + 1)
]
# Get script and options
script = "/home/fasig/scalable_radio_array/station_triggers_parallel.sh"
# Declare the error, output, log, and submit directories for Condor Job
error = '/data/user/fasig/pycondor'
output = '/data/user/fasig/pycondor'
log = '/data/user/fasig/pycondor'
submit = '/data/user/fasig/pycondor'
memory_requirement = str(int(args.stations / 100) + 1) + "GB"
# Setting up PyCondor Jobs
calculator_job = Job("calculate_" + basename,
script,
error=error,
output=output,
log=log,
submit=submit,
verbose=2,
request_memory=memory_requirement)
culminator_job = Job("culminate_" + basename,
script,
error=error,
output=output,
log=log,
submit=submit,
verbose=2)
# Add arguments to jobs
dirname = os.path.dirname(args.noise_file_basename)
if dirname == "":
dirname = "."
o = args.args[output_index]
replaced_name = logfile_name.replace("OUTPUT", o[:o.rindex(".")])
replaced_name = replaced_name.replace("ENERGY", energy)
replaced_name = replaced_name.replace("ITERATION",
str(i).zfill(zfill_amount))
args.args[logfile_index] = replaced_name
transfer_files.append(replaced_name)
file_remaps.append(replaced_name + '=' +
os.path.join(logfile_dirname, replaced_name))
job = Job(
descriptive_name + "_" + energy + "_" + str(i).zfill(zfill_amount),
executable=script_file,
output=output,
error=error,
log=log,
submit=submit, #request_memory="5GB",
extra_lines=[
"should_transfer_files = YES",
"transfer_output_files = " + ", ".join(transfer_files),
'transfer_output_remaps = "' + '; '.join(file_remaps) + '"',
"when_to_transfer_output = ON_EXIT"
],
verbose=2 if args.verbose else 0)
job.add_arg(" ".join([energy] + args.args))
dag.add_job(job)
# Write all necessary submit files and submit dagman to Condor
if args.maxjobs > 0:
dag.build_submit(submit_options="-maxjobs " + str(args.maxjobs))
else:
dag.build_submit()
def generate_dag(times, flags=[], tag='gwdetchar-omega-batch',
submit=False, outdir=os.getcwd(), universe='vanilla',
condor_commands=get_condor_arguments()):
"""Construct a Directed Acyclic Graph (DAG) for a batch of omega scans
Parameters
----------
times : `list` of `float`
list of GPS times to scan
flags : `list` of `str`, optional
a list of command line flags to run for each job, defaults to an
empty list
tag : `str`, optional
a helpful string to use to name the DAG,
default: `'gwdetchar-omega-batch'`
submit : `bool`, optional
submit the DAG to condor, default: `False`
outdir : `str`, optional
the output directory in which to store files, will result in
sub-directories called `'condor'` and `'log'`, default: `os.getcwd`
universe : `str`, optional
condor universe to run in, default: `'vanilla'`
condor_commands : `list` of `str`, optional
list of condor settings to process with, defaults to the output of
`get_condor_arguments`
Returns
-------
dagman : `~pycondor.Dagman`
the fully built DAG object
"""
logdir = os.path.join(outdir, 'logs')
subdir = os.path.join(outdir, 'condor')
executable = find_executable('gwdetchar-omega')
# create DAG and jobs
dagman = Dagman(name=tag, submit=subdir)
job = Job(
dag=dagman,
name=os.path.basename(executable),
executable=executable,
universe=universe,
submit=subdir,
error=logdir,
output=logdir,
getenv=True,
request_memory=4096 if universe != "local" else None,
extra_lines=condor_commands
)
# make a node in the workflow for each event time
for t in times:
cmd = " ".join([str(t)] + [
"--output-directory", os.path.join(outdir, str(t))] + flags)
job.add_arg(cmd, name=str(t).replace(".", "_"))
# write and submit the DAG
dagman.build(fancyname=False)
print("Workflow generated for {} times".format(len(times)))
if submit:
dagman.submit_dag(submit_options="-force")
print(
"Submitted to condor, check status via:\n\n"
"$ condor_q {}".format(getuser())
)
else:
print(
"Submit to condor via:\n\n"
"$ condor_submit_dag {0.submit_file}".format(dagman),
)
return dagman
def test_job_default_param_future_warning():
future_msg = ('The default values for the universe, getenv, and '
'notification Job parameters will be changed to None '
'in release version 0.5.0.')
with pytest.warns(FutureWarning, match=future_msg):
Job(name='jobname', executable=example_script)
def test_init_retry_type_fail():
with pytest.raises(TypeError) as excinfo:
job_with_retry = Job('jobname', example_script, retry='20')
job_with_retry.build()
error = 'retry must be an int'
assert error == str(excinfo.value)
def test_init_arguments_iterable():
arguments = ['arg{}'.format(i) for i in range(10)]
job = Job(name='jobname', executable=example_script, arguments=arguments)
assert len(job.args) == len(arguments)
for jobarg, argument in zip(job.args, arguments):
assert jobarg.arg == argument
def test_init_arguments():
arguments = 'my special argument'
job = Job(name='jobname', executable=example_script, arguments=arguments)
assert len(job.args) == 1
assert job.args[0].arg == arguments
