def test_convert_to_d_and_load(self):
import pbsmrtpipe.loader as L
rtasks, rfile_types, chunk_operators, pipelines = L.load_all()
for pipeline in pipelines.values():
pipeline_d = IO.pipeline_template_to_dict(pipeline, rtasks)
pipeline_loaded = IO.load_pipeline_template_from(pipeline_d)
self.assertEqual(pipeline.idx, pipeline_loaded.idx)
self.assertEqual(pipeline.display_name, pipeline_loaded.display_name)
self.assertEqual(len(pipeline.all_bindings), len(pipeline_loaded.all_bindings))
self.assertEqual(len(pipeline.entry_bindings), len(pipeline_loaded.entry_bindings))
def run_show_templates(output_dir=None):
import pbsmrtpipe.loader as L
from pbsmrtpipe.pb_io import write_pipeline_templates_to_avro
rtasks_d, _, _, pts = L.load_all()
print pretty_registered_pipelines(pts)
if output_dir is not None:
write_pipeline_templates_to_avro(pts.values(), rtasks_d, output_dir)
return 0
def __dynamically_load_all():
""" Load the registered tasks and operators
"""
import pbsmrtpipe.loader as L
def _f(x):
"""length or None"""
return "None" if x is None else len(x)
rtasks, rfile_types, roperators, rpipelines = L.load_all()
_d = dict(n=_f(rtasks), f=_f(rfile_types), o=_f(roperators), p=_f(rpipelines))
print "Registry Loaded. Number of ToolContracts:{n} FileTypes:{f} ChunkOperators:{o} Pipelines:{p}".format(**_d)
return rtasks, rfile_types, roperators, rpipelines
def validate_preset_xml(dir_name):
from pbsmrtpipe.pb_io import parse_pipeline_preset_xml, validate_raw_task_options
import pbsmrtpipe.loader as L
rtasks_d, _, _, pts = L.load_all()
for file_name in os.listdir(dir_name):
if file_name.endswith(".xml"):
p = parse_pipeline_preset_xml(op.join(dir_name, file_name))
if p.pipeline_id is None:
raise ValueError("{f} does not have pipeline-id set".format(
f=file_name))
elif not p.pipeline_id in pts:
raise ValueError("pipeline-id {i} not recognized".format(
i=p.pipeline_id))
log.info("validating {f}...".format(f=file_name))
validate_raw_task_options(rtasks_d, dict(p.task_options))
else:
log.warn("Skipping non-XML file {f}".format(f=file_name))
return 0
def test_convert_to_d_and_load(self):
import pbsmrtpipe.loader as L
rtasks, rfile_types, chunk_operators, pipelines = L.load_all()
for pipeline in pipelines.values():
pipeline_d = IO.pipeline_template_to_dict(pipeline, rtasks)
pipeline_loaded = IO.load_pipeline_template_from(pipeline_d)
self.assertEqual(pipeline.idx, pipeline_loaded.idx)
self.assertEqual(pipeline.display_name, pipeline_loaded.display_name)
self.assertEqual(len(pipeline.all_bindings), len(pipeline_loaded.all_bindings))
self.assertEqual(len(pipeline.entry_bindings), len(pipeline_loaded.entry_bindings))
# note that the internally registered pipeline does not necessarily
# have any task_options at this point, so we can't simply test for
# equality. however after another cycle they should be identical
if len(pipeline.task_options) > 0:
self.assertGreater(len(pipeline_loaded.task_options), 0)
pipeline_d2 = IO.pipeline_template_to_dict(pipeline_loaded, rtasks)
pipeline_loaded2 = IO.load_pipeline_template_from(pipeline_d2)
self.assertEqual(len(pipeline_loaded.task_options),
len(pipeline_loaded2.task_options))
for k,v in pipeline.task_options.iteritems():
if k in pipeline_loaded2.task_options:
self.assertEqual(v, pipeline_loaded2.task_options[k])
import logging
import os.path as op
import os
import sys
from pbcommand.models import (FileTypes, DataStoreViewRule,
PipelineDataStoreViewRules)
from pbsmrtpipe.loader import load_all
log = logging.getLogger(__name__)
SMRTLINK_VERSION = "5.1.0"
REGISTERED_VIEW_RULES = {}
RTASKS, _, _, PIPELINES = load_all()
def _to_view_rule(args):
return DataStoreViewRule(*args)
def __to_view_rule_triples(rules, is_hidden):
assert all([len(rule)==2 for rule in rules])
return [(rule[0], rule[1], is_hidden) for rule in rules]
def _to_whitelist(rules):
"""
Shorthand for whitelisting files given just the source IDs and file types.
Not suitable for cases where we need to override the label or description.
def test_sanity_load_installed_operators(self):
import pbsmrtpipe.loader as L
rtasks, rfile_types, chunk_operators, pipelines = L.load_all()
self.assertTrue(len(chunk_operators) > 0)
import argparse
import logging
import os.path as op
import os
import sys
from pbcommand.models import (FileTypes, DataStoreViewRule,
PipelineDataStoreViewRules)
from pbsmrtpipe.loader import load_all
log = logging.getLogger(__name__)
SMRTLINK_VERSION = "5.1.0"
REGISTERED_VIEW_RULES = {}
RTASKS, _, _, PIPELINES = load_all()
def _to_view_rule(args):
return DataStoreViewRule(*args)
def __to_view_rule_triples(rules, is_hidden):
assert all([len(rule) == 2 for rule in rules])
return [(rule[0], rule[1], is_hidden) for rule in rules]
def _to_whitelist(rules):
"""
Shorthand for whitelisting files given just the source IDs and file types.
Not suitable for cases where we need to override the label or description.