def test_output_up_to_date_func_changed(self):
"""Input file exists, output up to date, function body changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([transform1],
verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS,
pipeline="main")
# simulate source change
if sys.hexversion >= 0x03000000:
split1.__code__, transform1.__code__ = transform1.__code__, split1.__code__
else:
split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1],
verbose=6,
checksum_level=chksm,
pipeline="main")
if chksm >= CHECKSUM_FUNCTIONS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('Pipeline function has changed', s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
# clean up our function-changing mess!
if sys.hexversion >= 0x03000000:
split1.__code__, transform1.__code__ = transform1.__code__, split1.__code__
else:
split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
def test_transform_with_missing_formatter_args(self):
s = StringIO()
pipeline_printout(s, [transform_with_missing_formatter_args],
verbose=4, wrap_width=10000, pipeline="main")
self.assertIn("Unmatched field {dynamic_message}", s.getvalue())
pipeline_run([transform_with_missing_formatter_args],
verbose=0, pipeline="main")
def test_mkdir_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [check_transform, check_transform2],
verbose=5, wrap_width=10000, pipeline="main")
def test_raises_error(self):
"""run a function that fails but creates output, then check what should run"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
time.sleep(.5)
del runtime_data[:]
# poo. Shouldn't this be RuntimeError?
with self.assertRaises(RethrownJobError):
# generates output then fails
pipeline_run([transform_raise_error], verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS, pipeline="main")
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform_raise_error],
verbose=6, checksum_level=chksm, pipeline="main")
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('left over from a failed run?',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
def test_job_history_with_exceptions_run(self):
"""Run"""
for i in range(1):
cleanup_tmpdir()
try:
pipeline_run(
[test_task4],
verbose=0,
#multithread = 2,
one_second_per_job=one_second_per_job,
pipeline="main")
except:
pass
s = StringIO()
pipeline_printout(s, [test_task4],
verbose=VERBOSITY,
wrap_width=10000,
pipeline="main")
#
# task 2 should be up to date because exception was throw in task 3
#
pipeline_printout_str = s.getvalue()
correct_order = not re.search(
'Tasks which will be run:.*\n(.*\n)*Task = test_task2',
pipeline_printout_str)
if not correct_order:
print(pipeline_printout_str)
self.assertTrue(correct_order)
sys.stderr.write(".")
print()
def test_job_history_with_exceptions(self):
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_task4],
verbose=VERBOSITY,
wrap_width=10000,
pipeline="main")
def ruffus_main(options, args):
'Main entry point for ruffus pipelines'
if options.just_print:
pipeline_printout(
sys.stdout,
options.target_tasks,
options.forced_tasks,
verbose=options.verbose)
elif options.flowchart:
pipeline_printout_graph(
open(options.flowchart, "w"),
# use flowchart file name extension to decide flowchart format
# e.g. svg, jpg etc.
os.path.splitext(options.flowchart)[1][1:],
options.target_tasks,
options.forced_tasks,
no_key_legend=not options.key_legend_in_graph)
else:
pipeline_run(
options.target_tasks,
options.forced_tasks,
multiprocess=options.jobs,
logger=main_logger,
verbose=options.verbose,
touch_files_only=options.touch_only)
def test_merge_output(self):
"""test multiple-input checksums"""
# one output incorrectly generated
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([split1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
job_history = dbdict.open(get_default_history_file_name(), picklevalues=True)
del job_history[os.path.relpath(split1_outputs[0])]
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [merge2], verbose=5, checksum_level=chksm)
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('Previous incomplete run leftover', s.getvalue())
else:
self.assertIn('Job up-to-date', s.getvalue())
# make sure the jobs run fine
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([merge2], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [merge2], verbose=5, checksum_level=chksm)
self.assertIn('Job up-to-date', s.getvalue())
self.assertNotIn('Job needs update:', s.getvalue())
self.assertNotIn('Previous incomplete run leftover', s.getvalue())
def test_raises_error(self):
"""run a function that fails but creates output, then check what should run"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
time.sleep(.5)
del runtime_data[:]
# poo. Shouldn't this be RuntimeError?
with self.assertRaises(RethrownJobError):
# generates output then fails
pipeline_run([transform_raise_error],
verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS,
pipeline="main")
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform_raise_error],
verbose=6,
checksum_level=chksm,
pipeline="main")
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('left over from a failed run?', s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
def test_output_up_to_date_func_changed(self):
"""Input file exists, output up to date, function body changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([transform1], verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS, pipeline="main")
# simulate source change
if sys.hexversion >= 0x03000000:
split1.__code__, transform1.__code__ = transform1.__code__, split1.__code__
else:
split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=6,
checksum_level=chksm, pipeline="main")
if chksm >= CHECKSUM_FUNCTIONS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('Pipeline function has changed',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
# clean up our function-changing mess!
if sys.hexversion >= 0x03000000:
split1.__code__, transform1.__code__ = transform1.__code__, split1.__code__
else:
split1.func_code, transform1.func_code = transform1.func_code, split1.func_code
def test_ruffus (self):
print("\n\n Run pipeline normally...")
pipeline_run(multiprocess = 10, verbose=0, pipeline= "main")
check_final_output_correct()
check_job_order_correct(tempdir + "jobs.start")
check_job_order_correct(tempdir + "jobs.finish")
print(" OK")
print("\n\n Touch task2 only:")
os.unlink(os.path.join(tempdir, "jobs.start") )
os.unlink(os.path.join(tempdir, "jobs.finish") )
print(" First delete b.1 for task2...")
os.unlink(os.path.join(tempdir, "b.1"))
print(" Then run with touch_file_only...")
pipeline_run([task2], multiprocess = 10, touch_files_only=True, verbose = 0, pipeline= "main")
# check touching has made task2 up to date
s = StringIO()
pipeline_printout(s, [task2], verbose=4, wrap_width = 10000, pipeline= "main")
output_str = s.getvalue()
#print (">>>\n", output_str, "<<<\n", file=sys.stderr)
if "b.1" in output_str:
raise Exception("Expected b.1 created by touching...")
if "b.2" in output_str:
raise Exception("Expected b.2 created by touching...")
print(" Touching has made task2 up to date...\n")
print(" Then run normally again...")
pipeline_run(multiprocess = 10, verbose=0, pipeline= "main")
check_final_output_correct(True)
check_job_order_correct(tempdir + "jobs.start")
check_job_order_correct(tempdir + "jobs.finish")
def test_collate(self):
self.cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [combine_results], verbose=5, wrap_width = 10000, pipeline= "main")
self.assertTrue(re.search('Job needs update:.*Missing files.*', s.getvalue(), re.DOTALL) is not None)
#print s.getvalue()
pipeline_run([combine_results], verbose=0, pipeline= "main")
def test_permutations2_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_permutations2_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search('\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
'.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue()))
def test_collate(self):
self.cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [combine_results], verbose=5, wrap_width = 10000)
self.assertTrue('Job needs update: Missing files\n' in s.getvalue())
#print s.getvalue()
pipeline_run([combine_results], verbose=0)
def test_suffix_unmatched_printout2(self):
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_suffix_unmatched_task2],
verbose=5,
wrap_width=10000)
self.assertIn(
"Warning: File match failure: File 'tmp_test_regex_error_messages/a_name.tmp1' does not match suffix",
s.getvalue())
def test_suffix_printout(self):
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_suffix_task], verbose=5, wrap_width=10000)
self.assertTrue(
re.search(
'Missing files\n\s+\[tmp_test_regex_error_messages/a_name.tmp1, tmp_test_regex_error_messages/a_name.tmp2',
s.getvalue()))
def test_product_misspelt_capture_error(self):
"""Misspelt named capture group
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_misspelt_capture_error_task], verbose=3, wrap_width = 10000)
self.assertIn("Warning: File match failure: Unmatched field 'FILEPART'", s.getvalue())
def test_collate(self):
self.cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [combine_results], verbose=5, wrap_width=10000)
self.assertTrue('Job needs update: Missing files\n' in s.getvalue())
#print s.getvalue()
pipeline_run([combine_results], verbose=0)
def test_combinations_with_replacement2_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_combinations_with_replacement2_merged_task], verbose=5, wrap_width = 10000, pipeline= "main")
self.assertTrue(re.search('\[.*{tempdir}/a_name.tmp1, '
'.*{tempdir}/b_name.tmp1, '
'.*{tempdir}/a_name.b_name.tmp2\]'.format(tempdir=tempdir), s.getvalue()))
def test_printout_abbreviated_path1(self):
"""Input file exists, output doesn't exist"""
for syntax in decorator_syntax, oop_syntax:
s = StringIO()
if syntax == oop_syntax:
test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 1)
else:
pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 1, wrap_width = 500, pipeline= "main")
ret = s.getvalue()
self.assertTrue("[[.../job1.a.start, test_verbosity/job1.b.start]" in ret)
def test_transform_with_missing_formatter_args(self):
s = StringIO()
pipeline_printout(s, [transform_with_missing_formatter_args],
verbose=4,
wrap_width=10000,
pipeline="main")
self.assertIn("Unmatched field {dynamic_message}", s.getvalue())
pipeline_run([transform_with_missing_formatter_args],
verbose=0,
pipeline="main")
def test_product_misspelt_capture_error(self):
"""Misspelt named capture group
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_misspelt_capture_error_task], verbose=3, wrap_width = 10000, pipeline= "main")
self.assertIn("Warning: Input substitution failed:", s.getvalue())
self.assertIn("Unmatched field {FILEPART}", s.getvalue())
def test_product_formatter_ref_index_error(self):
"""
{path[0][0][1000} when len of the path string len(path[0][0]) < 1000
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_formatter_ref_index_error_task], verbose=3, wrap_width = 10000)
self.assertIn("Warning: File match failure: Unmatched field string index out of range", s.getvalue())
def test_product_out_of_range_formatter_ref_error(self):
"""
{path[2][0]} when len(path) == 1
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_out_of_range_formatter_ref_error_task], verbose=3, wrap_width = 10000)
self.assertIn("Warning: File match failure: Unmatched field 2L", s.getvalue())
def test_product_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_merged_task], verbose=5, wrap_width = 10000)
self.assertTrue(re.search('Job needs update: Missing files '
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/e_name.tmp1, '
'.*tmp_test_combinatorics/h_name.tmp1, '
'.*tmp_test_combinatorics/a_name.e_name.h_name.tmp2\]', s.getvalue()))
def test_combinations2_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_combinations2_merged_task], verbose=5, wrap_width = 10000, pipeline= "main")
self.assertTrue(re.search('Job needs update:.*Missing files.*'
'\[.*{tempdir}/a_name.tmp1, '
'.*{tempdir}/b_name.tmp1, '
'.*{tempdir}/a_name.b_name.tmp2\]'.format(tempdir=tempdir), s.getvalue(), re.DOTALL))
def test_product_out_of_range_formatter_ref_error(self):
"""
{path[2][0]} when len(path) == 1
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_out_of_range_formatter_ref_error_task], verbose=3, wrap_width = 10000, pipeline= "main")
self.assertIn("Warning: Input substitution failed:", s.getvalue())
self.assertIn("Unmatched field {2}", s.getvalue())
def test_printout_abbreviated_path0(self):
"""Input file exists, output doesn't exist"""
for syntax in decorator_syntax, oop_syntax:
s = StringIO()
if syntax == oop_syntax:
test_pipeline.printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 0, wrap_width = 500)
else:
pipeline_printout(s, [second_task], verbose = 5, verbose_abbreviated_path = 0, wrap_width = 500, pipeline= "main")
ret = s.getvalue()
path_str = os.path.abspath('%sdata/scratch/lg/what/one/two/three/four/five/six/seven/job2.a.start' % tempdir)
path_str = '[[%s' % path_str
def test_ouput_up_to_date(self):
"""Input file exists, output up to date"""
# output is up to date-- not run for any levels
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=5, checksum_level=chksm)
self.assertIn('Job up-to-date', s.getvalue())
def test_product_misspelt_capture_error(self):
"""Misspelt named capture group
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_misspelt_capture_error_task],
verbose=3,
wrap_width=10000)
self.assertIn(
"Warning: File match failure: Unmatched field 'FILEPART'",
s.getvalue())
def test_ouput_doesnt_exist(self):
"""Input file exists, output doesn't exist"""
# output doesn't exist-- should run for all levels
# create a new input file
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=5, checksum_level=chksm)
self.assertIn('Job needs update: Missing file [tmp_test_job_completion/input.output]',
s.getvalue())
def test_product_misspelt_capture_error(self):
"""Misspelt named capture group
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [check_product_misspelt_capture_error_task],
verbose=3,
wrap_width=10000,
pipeline="main")
self.assertIn("Warning: Input substitution failed:", s.getvalue())
self.assertIn("Unmatched field {FILEPART}", s.getvalue())
def test_product_formatter_ref_index_error(self):
"""
{path[0][0][1000} when len of the path string len(path[0][0]) < 1000
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [check_product_formatter_ref_index_error_task],
verbose=3, wrap_width=10000, pipeline="main")
self.assertIn("Warning: Input substitution failed:", s.getvalue())
self.assertIn(
"Unmatched field {string index out of range}", s.getvalue())
def test_combinations_with_replacement2_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_combinations_with_replacement2_merged_task],
verbose=5,
wrap_width=10000)
self.assertTrue(
re.search(
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/b_name.tmp1, '
'.*tmp_test_combinatorics/a_name.b_name.tmp2\]', s.getvalue()))
def test_product_formatter_ref_index_error(self):
"""
{path[0][0][1000} when len of the path string len(path[0][0]) < 1000
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_formatter_ref_index_error_task],
verbose=3,
wrap_width=10000)
self.assertIn(
"Warning: File match failure: Unmatched field string index out of range",
s.getvalue())
def test_collate(self):
self.cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [combine_results],
verbose=5,
wrap_width=10000,
pipeline="main")
self.assertTrue(
re.search('Job needs update:.*Missing files.*', s.getvalue(),
re.DOTALL) is not None)
# print s.getvalue()
pipeline_run([combine_results], verbose=0, pipeline="main")
def test_product_out_of_range_formatter_ref_error(self):
"""
{path[2][0]} when len(path) == 1
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s,
[test_product_out_of_range_formatter_ref_error_task],
verbose=3,
wrap_width=10000)
self.assertIn("Warning: File match failure: Unmatched field 2",
s.getvalue())
def test_product_formatter_ref_index_error(self):
"""
{path[0][0][1000} when len of the path string len(path[0][0]) < 1000
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [check_product_formatter_ref_index_error_task],
verbose=3,
wrap_width=10000,
pipeline="main")
self.assertIn("Warning: Input substitution failed:", s.getvalue())
self.assertIn("Unmatched field {string index out of range}",
s.getvalue())
def test_product_out_of_range_formatter_ref_error(self):
"""
{path[2][0]} when len(path) == 1
Requires verbose >= 3 or an empty jobs list
"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(
s, [check_product_out_of_range_formatter_ref_error_task],
verbose=3,
wrap_width=10000,
pipeline="main")
self.assertIn("Warning: Input substitution failed:", s.getvalue())
self.assertIn("Unmatched field {2}", s.getvalue())
def test_product_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [test_product_merged_task],
verbose=5,
wrap_width=10000)
self.assertTrue(
re.search(
'Job needs update: Missing files\n\s+'
'\[.*tmp_test_combinatorics/a_name.tmp1, '
'.*tmp_test_combinatorics/e_name.tmp1, '
'.*tmp_test_combinatorics/h_name.tmp1, '
'.*tmp_test_combinatorics/a_name.e_name.h_name.tmp2\]',
s.getvalue()))
def test_permutations2_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [check_permutations2_merged_task],
verbose=5,
wrap_width=10000,
pipeline="main")
self.assertTrue(
re.search(
'\[.*{tempdir}/a_name.tmp1, '
'.*{tempdir}/b_name.tmp1, '
'.*{tempdir}/a_name.b_name.tmp2\]'.format(tempdir=tempdir),
s.getvalue()))
def test_combinations2_printout(self):
"""Input file exists, output doesn't exist"""
cleanup_tmpdir()
s = StringIO()
pipeline_printout(s, [check_combinations2_merged_task],
verbose=5,
wrap_width=10000,
pipeline="main")
self.assertTrue(
re.search(
'Job needs update:.*Missing files.*'
'\[.*{tempdir}/a_name.tmp1, '
'.*{tempdir}/b_name.tmp1, '
'.*{tempdir}/a_name.b_name.tmp2\]'.format(tempdir=tempdir),
s.getvalue(), re.DOTALL))
def test_printout_abbreviated_path1(self):
"""Input file exists, output doesn't exist"""
for syntax in decorator_syntax, oop_syntax:
s = StringIO()
if syntax == oop_syntax:
test_pipeline.printout(s, [second_task],
verbose=5,
verbose_abbreviated_path=1)
else:
pipeline_printout(s, [second_task],
verbose=5,
verbose_abbreviated_path=1,
wrap_width=500,
pipeline="main")
ret = s.getvalue()
self.assertTrue(
"[[.../job1.a.start, test_verbosity/job1.b.start]" in ret)
def test_output_doesnt_exist(self):
"""Input file exists, output doesn't exist"""
# output doesn't exist-- should run for all levels
# create a new input file
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1],
verbose=6,
checksum_level=chksm,
pipeline="main")
self.assertTrue(
re.search(
r'Job needs update:.*Missing file.*\[tmp_test_job_completion/input.output\]',
s.getvalue(), re.DOTALL))
def test_ruffus (self):
# run first
pipeline_run(verbose = 0)
# should now be out of date
s = StringIO()
pipeline_printout(s, verbose = 5)
ret = s.getvalue()
try:
self.do_assertRegexpMatches(ret, r"Tasks which are up-to-date:(\n\s*)*Task = 'test_check_if_uptodate.task1'(\n\s*)*Task = 'test_check_if_uptodate.task2'")
except:
print ("\n\tOops: Both tasks should be up to date!!\n\n")
raise
try:
self.do_assertNotRegexpMatches(ret, r"Jobs needs update:\s*No function to check if up-to-date")
except:
print ("\n\tOops: @check_if_uptodate is not being picked up!!\n\n")
raise
def test_ouput_up_to_date_func_changed(self):
"""Input file exists, output up to date, function body changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
transform1.func_code = split1.func_code # simulate source change
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=5, checksum_level=chksm)
if chksm >= CHECKSUM_FUNCTIONS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('Pipeline function has changed',
s.getvalue())
else:
self.assertIn('Job up-to-date', s.getvalue())
def test_ouput_up_to_date_param_changed(self):
"""Input file exists, output up to date, parameter to function changed"""
# output is up to date, but function body changed (e.g., source different)
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([transform1], verbose=0, checksum_level=CHECKSUM_HISTORY_TIMESTAMPS)
runtime_data.append('different') # simulate change to config file
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=5, checksum_level=chksm)
if chksm >= CHECKSUM_FUNCTIONS_AND_PARAMS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('Pipeline parameters have changed',
s.getvalue())
else:
self.assertIn('Job up-to-date', s.getvalue())
def test_merge_output(self):
"""test multiple-input checksums"""
# one output incorrectly generated
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([split1],
verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS,
pipeline="main")
job_history = dbdict.open(get_default_history_file_name(),
picklevalues=True)
del job_history[os.path.relpath(split1_outputs[0])]
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [merge2],
verbose=6,
checksum_level=chksm,
pipeline="main")
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('left over from a failed run?', s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
# make sure the jobs run fine
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([merge2],
verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS,
pipeline="main")
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [merge2],
verbose=6,
checksum_level=chksm,
pipeline="main")
#self.assertIn('Job up-to-date', s.getvalue())
self.assertNotIn('Job needs update:', s.getvalue())
self.assertNotIn('left over from a failed run?', s.getvalue())
def test_ouput_timestamp_okay(self):
"""Input file exists, output timestamp up to date"""
# output exists and timestamp is up to date-- not run for lvl 0, run for all others
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
time.sleep(0.1)
with open(transform1_out, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=5, checksum_level=chksm)
if chksm == CHECKSUM_FILE_TIMESTAMPS:
self.assertIn('Job up-to-date', s.getvalue())
else:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('Previous incomplete run leftover',
s.getvalue())
def test_ouput_out_of_date(self):
"""Input file exists, output out of date"""
# output exists but is out of date-- should run for all levels
cleanup_tmpdir()
with open(transform1_out, 'w') as outfile:
outfile.write('testme')
time.sleep(0.1)
with open(input_file, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1], verbose=5, checksum_level=chksm)
self.assertIn('Job needs update:', s.getvalue())
if chksm == CHECKSUM_FILE_TIMESTAMPS:
self.assertIn('Input files:', s.getvalue())
self.assertIn('Output files:', s.getvalue())
else:
self.assertIn('Previous incomplete run leftover', s.getvalue())
def test_output_up_to_date(self):
"""Input file exists, output up to date"""
# output is up to date-- not run for any levels
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([transform1],
verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS,
pipeline="main")
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1],
verbose=6,
checksum_level=chksm,
pipeline="main")
#self.assertIn('Job up-to-date', s.getvalue())
pass
def test_printout_abbreviated_path0(self):
"""Input file exists, output doesn't exist"""
for syntax in decorator_syntax, oop_syntax:
s = StringIO()
if syntax == oop_syntax:
test_pipeline.printout(s, [second_task],
verbose=5,
verbose_abbreviated_path=0,
wrap_width=500)
else:
pipeline_printout(s, [second_task],
verbose=5,
verbose_abbreviated_path=0,
wrap_width=500,
pipeline="main")
ret = s.getvalue()
path_str = os.path.abspath(
'%sdata/scratch/lg/what/one/two/three/four/five/six/seven/job2.a.start'
% tempdir)
path_str = '[[%s' % path_str
def test_ruffus(self):
print("\n\n Run pipeline normally...")
pipeline_run(multiprocess=10, verbose=0, pipeline="main")
check_final_output_correct()
check_job_order_correct(tempdir + "jobs.start")
check_job_order_correct(tempdir + "jobs.finish")
print(" OK")
print("\n\n Touch task2 only:")
os.unlink(os.path.join(tempdir, "jobs.start"))
os.unlink(os.path.join(tempdir, "jobs.finish"))
print(" First delete b.1 for task2...")
os.unlink(os.path.join(tempdir, "b.1"))
print(" Then run with touch_file_only...")
pipeline_run([task2],
multiprocess=10,
touch_files_only=True,
verbose=0,
pipeline="main")
# check touching has made task2 up to date
s = StringIO()
pipeline_printout(s, [task2],
verbose=4,
wrap_width=10000,
pipeline="main")
output_str = s.getvalue()
#print (">>>\n", output_str, "<<<\n", file=sys.stderr)
if "b.1" in output_str:
raise Exception("Expected b.1 created by touching...")
if "b.2" in output_str:
raise Exception("Expected b.2 created by touching...")
print(" Touching has made task2 up to date...\n")
print(" Then run normally again...")
pipeline_run(multiprocess=10, verbose=0, pipeline="main")
check_final_output_correct(True)
check_job_order_correct(tempdir + "jobs.start")
check_job_order_correct(tempdir + "jobs.finish")
def test_split_output(self):
"""test multiple-output checksums"""
# outputs out of date
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([split1], verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS, pipeline="main")
time.sleep(.5)
with open(input_file, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [split1], verbose=6,
checksum_level=chksm, pipeline="main")
self.assertIn('Job needs update:', s.getvalue())
# all outputs incorrectly generated
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
time.sleep(.5)
for f in split1_outputs:
with open(f, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [split1], verbose=6,
checksum_level=chksm, pipeline="main")
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('left over from a failed run?',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
# one output incorrectly generated
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
pipeline_run([split1], verbose=0,
checksum_level=CHECKSUM_HISTORY_TIMESTAMPS, pipeline="main")
job_history = dbdict.open(
get_default_history_file_name(), picklevalues=True)
del job_history[os.path.relpath(split1_outputs[0])]
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [split1], verbose=6,
checksum_level=chksm, pipeline="main")
if chksm >= CHECKSUM_HISTORY_TIMESTAMPS:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('left over from a failed run?',
s.getvalue())
else:
#self.assertIn('Job up-to-date', s.getvalue())
pass
def test_job_history_with_exceptions_run(self):
"""Run"""
for i in range(1):
cleanup_tmpdir()
try:
pipeline_run([test_task4], verbose = 0,
#multithread = 2,
one_second_per_job = one_second_per_job)
except:
pass
s = StringIO()
pipeline_printout(s, [test_task4], verbose=5, wrap_width = 10000)
#
# task 2 should be up to date because exception was throw in task 3
#
pipeline_printout_str = s.getvalue()
correct_order = not re.search('Tasks which will be run:.*\n(.*\n)*Task = test_task2', pipeline_printout_str)
if not correct_order:
print(pipeline_printout_str)
self.assertTrue(correct_order)
sys.stderr.write(".")
print()
def test_output_out_of_date(self):
"""Input file exists, output out of date"""
# output exists but is out of date-- should run for all levels
cleanup_tmpdir()
with open(transform1_out, 'w') as outfile:
outfile.write('testme')
time.sleep(0.1)
with open(input_file, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1],
verbose=6,
checksum_level=chksm,
pipeline="main")
self.assertIn('Job needs update:', s.getvalue())
if chksm == CHECKSUM_FILE_TIMESTAMPS:
self.assertIn('Input files:', s.getvalue())
self.assertIn('Output files:', s.getvalue())
else:
self.assertIn('left over from a failed run?', s.getvalue())
def test_output_timestamp_okay(self):
"""Input file exists, output timestamp up to date"""
# output exists and timestamp is up to date-- not run for lvl 0, run for all others
cleanup_tmpdir()
with open(input_file, 'w') as outfile:
outfile.write('testme')
time.sleep(0.1)
with open(transform1_out, 'w') as outfile:
outfile.write('testme')
for chksm in possible_chksms:
s = StringIO()
pipeline_printout(s, [transform1],
verbose=6,
checksum_level=chksm,
pipeline="main")
if chksm == CHECKSUM_FILE_TIMESTAMPS:
#self.assertIn('Job up-to-date', s.getvalue())
pass
else:
self.assertIn('Job needs update:', s.getvalue())
self.assertIn('left over from a failed run?', s.getvalue())
def ruffus_main(options, args):
'Main entry point for ruffus pipelines'
if options.just_print:
pipeline_printout(sys.stdout,
options.target_tasks,
options.forced_tasks,
verbose=options.verbose)
elif options.flowchart:
pipeline_printout_graph(
open(options.flowchart, "w"),
# use flowchart file name extension to decide flowchart format
# e.g. svg, jpg etc.
os.path.splitext(options.flowchart)[1][1:],
options.target_tasks,
options.forced_tasks,
no_key_legend=not options.key_legend_in_graph)
else:
pipeline_run(options.target_tasks,
options.forced_tasks,
multiprocess=options.jobs,
logger=main_logger,
verbose=options.verbose,
touch_files_only=options.touch_only)
def test_ruffus(self):
# run first
pipeline_run(verbose=0)
# should now be out of date
s = StringIO()
pipeline_printout(s, verbose=5)
ret = s.getvalue()
try:
self.do_assertRegexpMatches(
ret,
r"Tasks which are up-to-date:(\n\s*)*Task = 'test_check_if_uptodate.task1'(\n\s*)*Task = 'test_check_if_uptodate.task2'"
)
except:
print("\n\tOops: Both tasks should be up to date!!\n\n")
raise
try:
self.do_assertNotRegexpMatches(
ret,
r"Jobs needs update:\s*No function to check if up-to-date")
except:
print("\n\tOops: @check_if_uptodate is not being picked up!!\n\n")
raise
