def test_convert_to_d_and_load(self):
import pbsmrtpipe.loader as L
import pprint
rtasks, rfile_types, chunk_operators, pipelines = L.load_all()
for pipeline in pipelines.values():
pipeline_d = IO.pipeline_template_to_dict(pipeline, rtasks)
#print "Raw Pipeline converted to dict"
#print pprint.pformat(pipeline_d)
pipeline_loaded = IO.load_pipeline_template_from(pipeline_d)
#print "Pipeline task options loaded from dict"
#print pprint.pformat(pipeline_loaded.task_options)
self.assertEqual(pipeline.idx, pipeline_loaded.idx)
self.assertEqual(pipeline.display_name,
pipeline_loaded.display_name)
self.assertEqual(len(pipeline.all_bindings),
len(pipeline_loaded.all_bindings))
self.assertEqual(len(pipeline.entry_bindings),
len(pipeline_loaded.entry_bindings))
# note that the internally registered pipeline does not necessarily
# have any task_options at this point, so we can't simply test for
# equality. however after another cycle they should be identical
if len(pipeline.task_options) > 0:
self.assertGreater(len(pipeline_loaded.task_options), 0)
pipeline_d2 = IO.pipeline_template_to_dict(
pipeline_loaded, rtasks)
#print pprint.pformat(pipeline_d)
pipeline_loaded2 = IO.load_pipeline_template_from(pipeline_d2)
self.assertEqual(len(pipeline_loaded.task_options),
len(pipeline_loaded2.task_options))
n = 1
for k, v in pipeline.task_options.iteritems():
if k in pipeline_loaded2.task_options:
v2 = pipeline_loaded2.task_options[k]
msg = "task option #{n} {k} expected '{v}' got '{x}'".format(
k=k, v=v, x=v2, n=n)
self.assertEqual(v, v2, msg)
print "Valid " + msg
n += 1
def test_all_sane(self):
"""Test that all pipelines are well defined"""
errors = []
rtasks, rfiles_types, chunk_operators, pipelines = L.load_all()
for pipeline_id, pipeline in pipelines.items():
emsg = "Pipeline {p} is not valid.".format(p=pipeline_id)
log.debug("Checking Sanity of registered Pipeline {i}".format(
i=pipeline_id))
log.info(pipeline_id)
log.debug(pipeline)
try:
# Validate with Avro
d = pipeline_template_to_dict(pipeline, rtasks)
_ = validate_pipeline_template(d)
name = pipeline_id + "_pipeline_template.avro"
output_file = get_temp_file(suffix=name)
log.info(
"{p} converted to avro successfully".format(p=pipeline_id))
bg = BU.binding_strs_to_binding_graph(rtasks,
pipeline.all_bindings)
BU.validate_binding_graph_integrity(bg)
BU.validate_compatible_binding_file_types(bg)
validate_entry_points(d)
# pprint.pprint(d)
# for debugging purposes
output_json = output_file.replace(".avro", '.json')
log.info("writing pipeline to {p}".format(p=output_json))
with open(output_json, 'w') as j:
j.write(json.dumps(d, sort_keys=True, indent=4))
log.info(
"writing pipeline template to {o}".format(o=output_file))
# Test writing to avro if the pipeline is actually valid
write_pipeline_template_to_avro(pipeline, rtasks, output_file)
log.info("Pipeline {p} is valid.".format(p=pipeline_id))
log.info("Loading avro {i} from {p}".format(i=pipeline_id,
p=output_file))
pipeline_d = load_pipeline_template_from_avro(output_file)
self.assertIsInstance(pipeline_d, dict)
except Exception as e:
m = emsg + " Error: " + e.message
log.error(m)
errors.append(m)
log.error(emsg)
log.error(e)
msg = "\n".join(errors) if errors else ""
self.assertEqual([], errors, msg)
def test_convert_to_d_and_load(self):
import pbsmrtpipe.loader as L
rtasks, rfile_types, chunk_operators, pipelines = L.load_all()
for pipeline in pipelines.values():
pipeline_d = IO.pipeline_template_to_dict(pipeline, rtasks)
pipeline_loaded = IO.load_pipeline_template_from(pipeline_d)
self.assertEqual(pipeline.idx, pipeline_loaded.idx)
self.assertEqual(pipeline.display_name, pipeline_loaded.display_name)
self.assertEqual(len(pipeline.all_bindings), len(pipeline_loaded.all_bindings))
self.assertEqual(len(pipeline.entry_bindings), len(pipeline_loaded.entry_bindings))
def test_convert_to_d_and_load(self):
import pbsmrtpipe.loader as L
rtasks, rfile_types, chunk_operators, pipelines = L.load_all()
for pipeline in pipelines.values():
pipeline_d = IO.pipeline_template_to_dict(pipeline, rtasks)
pipeline_loaded = IO.load_pipeline_template_from(pipeline_d)
self.assertEqual(pipeline.idx, pipeline_loaded.idx)
self.assertEqual(pipeline.display_name, pipeline_loaded.display_name)
self.assertEqual(len(pipeline.all_bindings), len(pipeline_loaded.all_bindings))
self.assertEqual(len(pipeline.entry_bindings), len(pipeline_loaded.entry_bindings))
# note that the internally registered pipeline does not necessarily
# have any task_options at this point, so we can't simply test for
# equality. however after another cycle they should be identical
if len(pipeline.task_options) > 0:
self.assertGreater(len(pipeline_loaded.task_options), 0)
pipeline_d2 = IO.pipeline_template_to_dict(pipeline_loaded, rtasks)
pipeline_loaded2 = IO.load_pipeline_template_from(pipeline_d2)
self.assertEqual(len(pipeline_loaded.task_options),
len(pipeline_loaded2.task_options))
for k,v in pipeline.task_options.iteritems():
if k in pipeline_loaded2.task_options:
self.assertEqual(v, pipeline_loaded2.task_options[k])