def dev_register(relative_id, display_name, tags=(), task_options=None):
pipeline_id = to_pipeline_ns(relative_id)
ptags = list(set(tags + (Tags.DEV, )))
return register_pipeline(pipeline_id,
display_name,
"0.1.0",
tags=ptags,
task_options=task_options)
def sa3_register(relative_id,
display_name,
version,
tags=(),
task_options=None):
pipeline_id = to_pipeline_ns(relative_id)
return register_pipeline(pipeline_id,
display_name,
version,
tags=tags,
task_options=task_options)
def dev_register(relative_id, display_name, tags=()):
pipeline_id = to_pipeline_ns(relative_id)
ptags = list(set(tags + (Tags.DENOVO, )))
return register_pipeline(pipeline_id, display_name, "0.1.0", tags=ptags)
def dev_register(relative_id, display_name, tags=()):
pipeline_id = to_pipeline_ns(relative_id)
ptags = list(set(tags + ('dev', )))
return register_pipeline(pipeline_id, display_name, "0.1.0", tags=ptags)
# Consensus Reports - variants
b4 = [(reference_ds, "pbreports.tasks.variants_report:0"),
("genomic_consensus.tasks.summarize_consensus:0",
"pbreports.tasks.variants_report:1"),
("genomic_consensus.tasks.variantcaller:0",
"pbreports.tasks.variants_report:2")]
# Consensus Reports - top variants
b5 = [("genomic_consensus.tasks.variantcaller:0",
"pbreports.tasks.top_variants:0"),
(reference_ds, "pbreports.tasks.top_variants:1")]
return b1 + b2 + b3 + b4 + b5
@register_pipeline(to_pipeline_ns("sa3_fetch"),
"RS Movie to Subread DataSet",
"0.1.0",
tags=("convert", ))
def sa3_fetch():
"""
SA3 Convert RS movie metadata XML to Subread DataSet XML
"""
# convert to RS dataset
b1 = [(Constants.ENTRY_RS_MOVIE_XML, "pbscala.tasks.rs_movie_to_ds_rtc:0")]
b2 = [("pbscala.tasks.rs_movie_to_ds_rtc:0",
"pbsmrtpipe.tasks.h5_subreads_to_subread:0")]
return b1 + b2
def sa3_register(relative_id, display_name, version, tags=(), task_options=None):
pipeline_id = to_pipeline_ns(relative_id)
return register_pipeline(pipeline_id, display_name, version, tags=tags, task_options=task_options)
def dev_register(relative_id, display_name, tags=(), task_options=None):
pipeline_id = to_pipeline_ns(relative_id)
ptags = list(set(tags + (Tags.DEV, )))
return register_pipeline(pipeline_id, display_name, "0.1.0", tags=ptags, task_options=task_options)
b3 = [("pbreports.tasks.summarize_coverage:0", "genomic_consensus.tasks.summarize_consensus:0"),
("genomic_consensus.tasks.variantcaller:0", "genomic_consensus.tasks.summarize_consensus:1")]
# Consensus Reports - variants
b4 = [(reference_ds, "pbreports.tasks.variants_report:0"),
("genomic_consensus.tasks.summarize_consensus:0", "pbreports.tasks.variants_report:1"),
("genomic_consensus.tasks.variantcaller:0", "pbreports.tasks.variants_report:2")]
# Consensus Reports - top variants
b5 = [("genomic_consensus.tasks.variantcaller:0", "pbreports.tasks.top_variants:0"),
(reference_ds, "pbreports.tasks.top_variants:1")]
return b1 + b2 + b3 + b4 + b5
@register_pipeline(to_pipeline_ns("sa3_fetch"), "RS Movie to Subread DataSet", "0.1.0", tags=("convert", ))
def sa3_fetch():
"""
SA3 Convert RS movie metadata XML to Subread DataSet XML
"""
# convert to RS dataset
b1 = [(Constants.ENTRY_RS_MOVIE_XML, "pbscala.tasks.rs_movie_to_ds_rtc:0")]
b2 = [("pbscala.tasks.rs_movie_to_ds_rtc:0", "pbsmrtpipe.tasks.h5_subreads_to_subread:0")]
return b1 + b2
@register_pipeline(to_pipeline_ns("sa3_align"), "SA3 RS movie Align", "0.1.0", tags=("mapping", ))
def sa3_align():
def register(relative_id, display_name, tags=(), task_options=None):
pipeline_id = to_pipeline_ns(relative_id)
ptags = list(set(tags + (Tags.INTERNAL, Tags.COND)))
return register_pipeline(pipeline_id, display_name, "0.2.0", tags=ptags, task_options=task_options)
class _TestBase(unittest.TestCase):
NTASKS = 10
NFILES = 12
EPOINTS = 1
PB_PIPELINE_ID = to_pipeline_ns("rs_fetch_1")
# this will be created from EPOINTS_NAMES in class setup
EPOINTS_D = {}
EPOINTS_NAMES = {'eid_input_xml': "_entry_point.xml"}
TASK_OPTIONS = {to_opt_id('filter_artifact_score'): -7,
to_opt_id('filter_max_read_length'): 10000,
to_opt_id('filter_min_read_length'): 1000}
# this object could create manually
PRESET_XML = 'cli_preset_01.xml'
@classmethod
def setUpClass(cls):
pipeline = REGISTERED_PIPELINES[cls.PB_PIPELINE_ID]
log.debug(pipeline)
cls.bindings = pipeline.all_bindings
cls.EPOINTS_D = {k: get_temp_file(v) for k, v in cls.EPOINTS_NAMES.iteritems()}
log.debug(pprint.pformat(cls.bindings, indent=4))
log.debug("Number of registered tasks {n}".format(n=len(REGISTERED_TASKS)))
cls.bgraph = B.binding_strs_to_binding_graph(REGISTERED_TASKS, cls.bindings)
d = os.path.expanduser('~/scratch/tmp_pbsmrtpipe') if getpass.getuser() == 'mkocher' else None
cls.output_dir = tempfile.mkdtemp(prefix='job_test_', dir=d)
preset_record = IO.parse_pipeline_preset_xml(os.path.join(TEST_DATA_DIR, cls.PRESET_XML))
cls.workflow_options = preset_record.to_workflow_level_opt()
# leave this for now
cls.envs = []
cls.cluster_engine = C.load_installed_cluster_templates_by_name("sge")
@classmethod
def tearDownClass(cls):
if not DEEP_DEBUG:
if hasattr(cls, 'output_dir'):
if os.path.exists(cls.output_dir):
shutil.rmtree(cls.output_dir)
def test_validate_bindings_graph(self):
emsg = "Invalid workflow with id '{x}'".format(x=self.PB_PIPELINE_ID)
self.assertTrue(B.validate_binding_graph_integrity(self.bgraph), emsg)
def test_number_of_entry_point_nodes(self):
"""Basic running test"""
n = len(self.bgraph.entry_point_nodes())
self.assertEqual(self.EPOINTS, n)
def test_number_of_tasks(self):
n = len(self.bgraph.task_nodes())
self.assertEqual(self.NTASKS, n)
def test_is_validate_binding_graph(self):
self.assertTrue(B.validate_binding_graph_integrity(self.bgraph))
def test_is_validate_binding_types(self):
self.assertTrue(B.validate_compatible_binding_file_types(self.bgraph))
def test_n_files(self):
self.assertEqual(self.NFILES, len(self.bgraph.file_nodes()))
@unittest.skip
def test_mock_runner(self):
B.resolve_entry_points(self.bgraph, self.EPOINTS_D)
state = M.mock_workflow_runner(self.bgraph, {},
self.output_dir,
self.workflow_options,
self.TASK_OPTIONS,
REGISTERED_FILE_TYPES,
self.cluster_engine, self.envs)
_ = B.get_tasks_by_state(self.bgraph, B.TaskStates.SUCCESSFUL)
if state is False:
log.debug(B.to_binding_graph_summary(self.bgraph))
self.assertTrue(state)
b3 = [("pbreports.tasks.summarize_coverage:0", "genomic_consensus.tasks.summarize_consensus:0"),
("genomic_consensus.tasks.variantcaller:0", "genomic_consensus.tasks.summarize_consensus:1")]
# Consensus Reports - variants
b4 = [(reference_ds, "pbreports.tasks.variants_report:0"),
("genomic_consensus.tasks.summarize_consensus:0", "pbreports.tasks.variants_report:1"),
("genomic_consensus.tasks.variantcaller:0", "pbreports.tasks.variants_report:2")]
# Consensus Reports - top variants
b5 = [("genomic_consensus.tasks.variantcaller:0", "pbreports.tasks.top_variants:0"),
(reference_ds, "pbreports.tasks.top_variants:1")]
return b1 + b2 + b3 + b4 + b5
@register_pipeline(to_pipeline_ns("sa3_fetch"), "RS Movie to Subread DataSet", "0.1.0", tags=("convert", ))
def sa3_fetch():
"""
Convert RS movie metadata XML to Subread DataSet XML
"""
# convert to RS dataset
b1 = [(Constants.ENTRY_RS_MOVIE_XML, "pbscala.tasks.rs_movie_to_ds_rtc:0")]
b2 = [("pbscala.tasks.rs_movie_to_ds_rtc:0", "pbsmrtpipe.tasks.h5_subreads_to_subread:0")]
return b1 + b2
@register_pipeline(to_pipeline_ns("sa3_align"), "SA3 RS movie Align", "0.1.0", tags=("mapping", ))
def sa3_align():
def register(relative_id, display_name, tags=(), task_options=None):
pipeline_id = to_pipeline_ns(relative_id)
ptags = list(set(tags + (Tags.INTERNAL, Tags.COND)))
return register_pipeline(pipeline_id, display_name, "0.2.0", tags=ptags, task_options=task_options)