def test_dagman_job_order(tmpdir):
# Test to check that the order in which Jobs are added to a Dagman doesn't
# change the Dagman submit file that is built. See issue #57.
submit_dir = str(tmpdir.mkdir('submit'))
dag_submit_lines = []
for order_idx in range(2):
dagman = Dagman('testdagman', submit=submit_dir)
job_child = Job('childjob', example_script, submit=submit_dir)
job_child.add_arg('--length 200', name='200jobname')
job_child.add_arg('--length 400', retry=3)
job_parent = Job('parentjob', example_script, submit=submit_dir)
job_parent.add_arg('--length 100')
job_parent.add_child(job_child)
if order_idx == 0:
# Add job_parent to dagman first
dagman.add_job(job_parent)
dagman.add_job(job_child)
else:
# Add job_child to dagman first
dagman.add_job(job_child)
dagman.add_job(job_parent)
dagman.build(fancyname=False)
# Append submit file lines to dag_submit_lines
with open(dagman.submit_file, 'r') as dag_submit_file:
dag_submit_lines.append(dag_submit_file.readlines())
# Test that the same lines occur in the Dagman submit file for
# adding the parent/child jobs in either order
assert Counter(dag_submit_lines[0]) == Counter(dag_submit_lines[1])
arguments += " --number " + str(args.number)
arguments += " --size " + str(args.size)
arguments += " --time " + str(args.time)
arguments += " --dt " + str(args.dt)
arguments += " --rms " + str(args.rms)
generator_job.add_arg(arguments)
if add_file_indices:
files = [
filename + "_" + suffix + ".npz" for suffix in output_suffixes
]
else:
files = [filename + ".npz"]
arguments = " ".join(files)
arguments += " --output " + str(args.envelope)
arguments += " --amplification " + str(args.amplification)
processor_job.add_arg(arguments)
# Create job dependencies
# processor_job doesn't start until generator_job has finished
generator_job.add_child(processor_job)
# Set up a dagman
dagman = Dagman("generate_" + basename, submit=submit, verbose=2)
# Add jobs to dagman
dagman.add_job(generator_job)
dagman.add_job(processor_job)
# Write all necessary submit files and submit job to Condor
dagman.build_submit()
for filename in sorted(os.listdir(dirname)):
filename = filename[:filename.rindex("_")]
if filename.startswith(basename) and not (filename in file_bases):
file_bases.append(filename)
for base in file_bases:
arguments = os.path.join(os.path.dirname(args.noise_file_basename), base)
arguments += " " + str(args.outfile)
arguments += " --range " + str(args.range[0]) + " " + str(args.range[1])
arguments += " --stations " + str(args.stations)
arguments += " --geometry " + str(args.geometry)
arguments += " --threshold " + str(args.threshold)
arguments += " --tot " + str(args.tot)
arguments += " --antennas_hit " + str(args.antennas_hit)
calculator_job.add_arg(arguments)
culminator_job.add_arg(args.outfile)
# Create job dependencies
# culminator_job doesn't start until calculator_job has finished
calculator_job.add_child(culminator_job)
# Set up a dagman
dagman = Dagman("full_calculation_" + basename, submit=submit, verbose=2)
# Add jobs to dagman
dagman.add_job(calculator_job)
dagman.add_job(culminator_job)
# Write all necessary submit files and submit job to Condor
dagman.build_submit()
