def _get_opts():
oid = OP.to_opt_id('dev.hello_message')
return {
oid:
OP.to_option_schema(oid, "string", 'Hello Message',
"Hello Message for dev Task.", "Default Message")
}
def _get_simple_opts():
# Util func to create an task option id 'pbsmrtpipe.task_options.dev.hello_message'
oid = OP.to_opt_id('dev.hello_message')
return {
oid:
OP.to_option_schema(oid, "string", 'Hello Message',
"Hello Message for dev Task.", "Default Message")
}
def _get_opts():
oid = OP.to_opt_id('dev.hello_message')
return {oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")}
def _get_simple_opts():
# Util func to create an task option id 'pbsmrtpipe.task_options.dev.hello_message'
oid = OP.to_opt_id('dev.hello_message')
return {oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")}
s = func()
_ = jsonschema.Draft4Validator(s)
oid = s['properties'].keys()[0]
REGISTERED_WORKFLOW_OPTIONS[oid] = s
return func
def register_validation_func(option_id):
def wrapper(func):
OPTION_VALIDATORS[option_id].append(func)
return wrapper
@register_validation_func(OP.to_opt_id('tmp_dir'))
def validator(value):
if os.path.isdir(value):
return value
raise ValueError(
"Option id '{i}' invalid. Unable to find {v}".format(v=value))
@register_workflow_option
def _to_max_chunks_option():
return OP.to_option_schema(
_to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks",
"Max Number of chunks that a file will be scattered into",
GlobalConstants.MAX_NCHUNKS)
import pbsmrtpipe.schema_opt_utils as OP
# generate a task option id
oid = OP.to_opt_id('dev.hello_message')
# generate a schema
s = OP.to_option_schema(oid, "string", 'Hello Message',
"Hello Message for dev Task.", "Default Message")
print {oid: s}
import logging
from pbsmrtpipe.schema_opt_utils import to_opt_id
log = logging.getLogger(__name__)
_FILTER_OPTS_NAMES = 'filter_trim filter_artifact_score use_subreads ' \
'filter_read_score filter_min_read_length ' \
'filter_max_read_length filter_min_subread_length ' \
'filter_max_subread_length ' \
'filter_whitelist filter_min_snr'.split()
_FILTER_OPTS = [to_opt_id(s) for s in _FILTER_OPTS_NAMES]
s = func()
_ = jsonschema.Draft4Validator(s)
oid = s["properties"].keys()[0]
REGISTERED_WORKFLOW_OPTIONS[oid] = s
return func
def register_validation_func(option_id):
def wrapper(func):
OPTION_VALIDATORS[option_id].append(func)
return wrapper
@register_validation_func(OP.to_opt_id("tmp_dir"))
def validator(value):
if os.path.isdir(value):
return value
raise ValueError("Option id '{i}' invalid. Unable to find {v}".format(v=value))
@register_workflow_option
def _to_max_chunks_option():
return OP.to_option_schema(
_to_wopt_id("max_nchunks"),
"integer",
"Max Number of Chunks",
"Max Number of chunks that a file will be scattered into",
GlobalConstants.MAX_NCHUNKS,
)
s = func()
_ = jsonschema.Draft4Validator(s)
oid = s['properties'].keys()[0]
REGISTERED_WORKFLOW_OPTIONS[oid] = s
return func
def register_validation_func(option_id):
def wrapper(func):
OPTION_VALIDATORS[option_id].append(func)
return wrapper
@register_validation_func(OP.to_opt_id('tmp_dir'))
def validator(value):
if os.path.isdir(value):
return value
raise ValueError("Option id '{i}' invalid. Unable to find {v}".format(v=value))
@register_workflow_option
def _to_max_chunks_option():
return OP.to_option_schema(_to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks",
"Max Number of chunks that a file will be scattered into", GlobalConstants.MAX_NCHUNKS)
@register_workflow_option
def _to_max_nproc_option():
return OP.to_option_schema(_to_wopt_id("max_nproc"), "integer",
class TestTestDevSimpleTask(TestDevSimpleTaskDefaults):
# Test different set of options
TASK_OPTIONS = {to_opt_id('dev.hello_message'): "Custom Message"}
class _TestBase(unittest.TestCase):
NTASKS = 10
NFILES = 12
EPOINTS = 1
PB_PIPELINE_ID = to_pipeline_ns("rs_fetch_1")
# this will be created from EPOINTS_NAMES in class setup
EPOINTS_D = {}
EPOINTS_NAMES = {'eid_input_xml': "_entry_point.xml"}
TASK_OPTIONS = {to_opt_id('filter_artifact_score'): -7,
to_opt_id('filter_max_read_length'): 10000,
to_opt_id('filter_min_read_length'): 1000}
# this object could create manually
PRESET_XML = 'cli_preset_01.xml'
@classmethod
def setUpClass(cls):
pipeline = REGISTERED_PIPELINES[cls.PB_PIPELINE_ID]
log.debug(pipeline)
cls.bindings = pipeline.all_bindings
cls.EPOINTS_D = {k: get_temp_file(v) for k, v in cls.EPOINTS_NAMES.iteritems()}
log.debug(pprint.pformat(cls.bindings, indent=4))
log.debug("Number of registered tasks {n}".format(n=len(REGISTERED_TASKS)))
cls.bgraph = B.binding_strs_to_binding_graph(REGISTERED_TASKS, cls.bindings)
d = os.path.expanduser('~/scratch/tmp_pbsmrtpipe') if getpass.getuser() == 'mkocher' else None
cls.output_dir = tempfile.mkdtemp(prefix='job_test_', dir=d)
preset_record = IO.parse_pipeline_preset_xml(os.path.join(TEST_DATA_DIR, cls.PRESET_XML))
cls.workflow_options = preset_record.to_workflow_level_opt()
# leave this for now
cls.envs = []
cls.cluster_engine = C.load_installed_cluster_templates_by_name("sge")
@classmethod
def tearDownClass(cls):
if not DEEP_DEBUG:
if hasattr(cls, 'output_dir'):
if os.path.exists(cls.output_dir):
shutil.rmtree(cls.output_dir)
def test_validate_bindings_graph(self):
emsg = "Invalid workflow with id '{x}'".format(x=self.PB_PIPELINE_ID)
self.assertTrue(B.validate_binding_graph_integrity(self.bgraph), emsg)
def test_number_of_entry_point_nodes(self):
"""Basic running test"""
n = len(self.bgraph.entry_point_nodes())
self.assertEqual(self.EPOINTS, n)
def test_number_of_tasks(self):
n = len(self.bgraph.task_nodes())
self.assertEqual(self.NTASKS, n)
def test_is_validate_binding_graph(self):
self.assertTrue(B.validate_binding_graph_integrity(self.bgraph))
def test_is_validate_binding_types(self):
self.assertTrue(B.validate_compatible_binding_file_types(self.bgraph))
def test_n_files(self):
self.assertEqual(self.NFILES, len(self.bgraph.file_nodes()))
@unittest.skip
def test_mock_runner(self):
B.resolve_entry_points(self.bgraph, self.EPOINTS_D)
state = M.mock_workflow_runner(self.bgraph, {},
self.output_dir,
self.workflow_options,
self.TASK_OPTIONS,
REGISTERED_FILE_TYPES,
self.cluster_engine, self.envs)
_ = B.get_tasks_by_state(self.bgraph, B.TaskStates.SUCCESSFUL)
if state is False:
log.debug(B.to_binding_graph_summary(self.bgraph))
self.assertTrue(state)
import pbsmrtpipe.schema_opt_utils as OP
# generate a task option id
oid = OP.to_opt_id('dev.hello_message')
# generate a schema
s = OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")
print {oid: s}