def _get_opts(): oid = OP.to_opt_id('dev.hello_message') return { oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message") }
def _get_simple_opts(): # Util func to create an task option id 'pbsmrtpipe.task_options.dev.hello_message' oid = OP.to_opt_id('dev.hello_message') return { oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message") }
def _get_opts(): oid = OP.to_opt_id('dev.hello_message') return {oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")}
def _get_simple_opts(): # Util func to create an task option id 'pbsmrtpipe.task_options.dev.hello_message' oid = OP.to_opt_id('dev.hello_message') return {oid: OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message")}
s = func() _ = jsonschema.Draft4Validator(s) oid = s['properties'].keys()[0] REGISTERED_WORKFLOW_OPTIONS[oid] = s return func def register_validation_func(option_id): def wrapper(func): OPTION_VALIDATORS[option_id].append(func) return wrapper @register_validation_func(OP.to_opt_id('tmp_dir')) def validator(value): if os.path.isdir(value): return value raise ValueError( "Option id '{i}' invalid. Unable to find {v}".format(v=value)) @register_workflow_option def _to_max_chunks_option(): return OP.to_option_schema( _to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks", "Max Number of chunks that a file will be scattered into", GlobalConstants.MAX_NCHUNKS)
import pbsmrtpipe.schema_opt_utils as OP # generate a task option id oid = OP.to_opt_id('dev.hello_message') # generate a schema s = OP.to_option_schema(oid, "string", 'Hello Message', "Hello Message for dev Task.", "Default Message") print {oid: s}
import logging from pbsmrtpipe.schema_opt_utils import to_opt_id log = logging.getLogger(__name__) _FILTER_OPTS_NAMES = 'filter_trim filter_artifact_score use_subreads ' \ 'filter_read_score filter_min_read_length ' \ 'filter_max_read_length filter_min_subread_length ' \ 'filter_max_subread_length ' \ 'filter_whitelist filter_min_snr'.split() _FILTER_OPTS = [to_opt_id(s) for s in _FILTER_OPTS_NAMES]
s = func() _ = jsonschema.Draft4Validator(s) oid = s["properties"].keys()[0] REGISTERED_WORKFLOW_OPTIONS[oid] = s return func def register_validation_func(option_id): def wrapper(func): OPTION_VALIDATORS[option_id].append(func) return wrapper @register_validation_func(OP.to_opt_id("tmp_dir")) def validator(value): if os.path.isdir(value): return value raise ValueError("Option id '{i}' invalid. Unable to find {v}".format(v=value)) @register_workflow_option def _to_max_chunks_option(): return OP.to_option_schema( _to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks", "Max Number of chunks that a file will be scattered into", GlobalConstants.MAX_NCHUNKS, )
s = func() _ = jsonschema.Draft4Validator(s) oid = s['properties'].keys()[0] REGISTERED_WORKFLOW_OPTIONS[oid] = s return func def register_validation_func(option_id): def wrapper(func): OPTION_VALIDATORS[option_id].append(func) return wrapper @register_validation_func(OP.to_opt_id('tmp_dir')) def validator(value): if os.path.isdir(value): return value raise ValueError("Option id '{i}' invalid. Unable to find {v}".format(v=value)) @register_workflow_option def _to_max_chunks_option(): return OP.to_option_schema(_to_wopt_id("max_nchunks"), "integer", "Max Number of Chunks", "Max Number of chunks that a file will be scattered into", GlobalConstants.MAX_NCHUNKS) @register_workflow_option def _to_max_nproc_option(): return OP.to_option_schema(_to_wopt_id("max_nproc"), "integer",
class TestTestDevSimpleTask(TestDevSimpleTaskDefaults): # Test different set of options TASK_OPTIONS = {to_opt_id('dev.hello_message'): "Custom Message"}
class _TestBase(unittest.TestCase): NTASKS = 10 NFILES = 12 EPOINTS = 1 PB_PIPELINE_ID = to_pipeline_ns("rs_fetch_1") # this will be created from EPOINTS_NAMES in class setup EPOINTS_D = {} EPOINTS_NAMES = {'eid_input_xml': "_entry_point.xml"} TASK_OPTIONS = {to_opt_id('filter_artifact_score'): -7, to_opt_id('filter_max_read_length'): 10000, to_opt_id('filter_min_read_length'): 1000} # this object could create manually PRESET_XML = 'cli_preset_01.xml' @classmethod def setUpClass(cls): pipeline = REGISTERED_PIPELINES[cls.PB_PIPELINE_ID] log.debug(pipeline) cls.bindings = pipeline.all_bindings cls.EPOINTS_D = {k: get_temp_file(v) for k, v in cls.EPOINTS_NAMES.iteritems()} log.debug(pprint.pformat(cls.bindings, indent=4)) log.debug("Number of registered tasks {n}".format(n=len(REGISTERED_TASKS))) cls.bgraph = B.binding_strs_to_binding_graph(REGISTERED_TASKS, cls.bindings) d = os.path.expanduser('~/scratch/tmp_pbsmrtpipe') if getpass.getuser() == 'mkocher' else None cls.output_dir = tempfile.mkdtemp(prefix='job_test_', dir=d) preset_record = IO.parse_pipeline_preset_xml(os.path.join(TEST_DATA_DIR, cls.PRESET_XML)) cls.workflow_options = preset_record.to_workflow_level_opt() # leave this for now cls.envs = [] cls.cluster_engine = C.load_installed_cluster_templates_by_name("sge") @classmethod def tearDownClass(cls): if not DEEP_DEBUG: if hasattr(cls, 'output_dir'): if os.path.exists(cls.output_dir): shutil.rmtree(cls.output_dir) def test_validate_bindings_graph(self): emsg = "Invalid workflow with id '{x}'".format(x=self.PB_PIPELINE_ID) self.assertTrue(B.validate_binding_graph_integrity(self.bgraph), emsg) def test_number_of_entry_point_nodes(self): """Basic running test""" n = len(self.bgraph.entry_point_nodes()) self.assertEqual(self.EPOINTS, n) def test_number_of_tasks(self): n = len(self.bgraph.task_nodes()) self.assertEqual(self.NTASKS, n) def test_is_validate_binding_graph(self): self.assertTrue(B.validate_binding_graph_integrity(self.bgraph)) def test_is_validate_binding_types(self): self.assertTrue(B.validate_compatible_binding_file_types(self.bgraph)) def test_n_files(self): self.assertEqual(self.NFILES, len(self.bgraph.file_nodes())) @unittest.skip def test_mock_runner(self): B.resolve_entry_points(self.bgraph, self.EPOINTS_D) state = M.mock_workflow_runner(self.bgraph, {}, self.output_dir, self.workflow_options, self.TASK_OPTIONS, REGISTERED_FILE_TYPES, self.cluster_engine, self.envs) _ = B.get_tasks_by_state(self.bgraph, B.TaskStates.SUCCESSFUL) if state is False: log.debug(B.to_binding_graph_summary(self.bgraph)) self.assertTrue(state)