def _load_io_for_task(registered_tasks, entry_points_d, preset_xml, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None): """Grungy loading of the IO and resolving values Returns a tuple of (WorkflowLevelOptions, TaskOptions, ClusterRender) """ slog.info("validating entry points. {e}".format(e=entry_points_d)) _validate_entry_points_or_raise(entry_points_d) slog.info("successfully validated {n} entry points".format(n=len(entry_points_d))) wopts = {} topts = {} if rc_preset_or_none is None: rc_preset = IO.load_preset_from_env() else: rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none) if rc_preset: topts.update(dict(rc_preset.task_options)) wopts.update(dict(rc_preset.workflow_options)) if preset_xml is not None: preset_record = IO.parse_pipeline_preset_xml(preset_xml) wopts.update(dict(preset_record.workflow_options)) topts.update(dict(preset_record.task_options)) workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts) workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts) if isinstance(force_chunk_mode, bool): workflow_level_opts.chunk_mode = force_chunk_mode # Validate topts = IO.validate_raw_task_options(registered_tasks, topts) log.debug("Resolved task options to {d}".format(d=workflow_level_opts)) log.debug(pprint.pprint(workflow_level_opts.to_dict(), indent=4)) if isinstance(workflow_level_opts.cluster_manager_path, str): cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path) # override distributed mode if isinstance(force_distribute, bool): workflow_level_opts.distributed_mode = force_distribute else: cluster_render = None workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS) if workflow_level_opts.distributed_mode is False: slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count())) workflow_level_opts.total_max_nproc = multiprocessing.cpu_count() if debug_mode is True: slog.info("overriding debug-mode to True") workflow_level_opts.debug_mode = debug_mode return workflow_level_opts, topts, cluster_render
def _load_io_for_task(registered_tasks, entry_points_d, preset_xmls, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None): """Grungy loading of the IO and resolving values Returns a tuple of (WorkflowLevelOptions, TaskOptions, ClusterRender) """ slog.info("validating entry points. {e}".format(e=entry_points_d)) _validate_entry_points_or_raise(entry_points_d) slog.info("successfully validated {n} entry points".format(n=len(entry_points_d))) wopts = {} topts = {} if rc_preset_or_none is None: rc_preset = IO.load_preset_from_env() else: rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none) if rc_preset: topts.update(dict(rc_preset.task_options)) wopts.update(dict(rc_preset.workflow_options)) if preset_xmls: preset_record = IO.parse_pipeline_preset_xmls(preset_xmls) wopts.update(dict(preset_record.workflow_options)) topts.update(dict(preset_record.task_options)) workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts) workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts) if isinstance(force_chunk_mode, bool): workflow_level_opts.chunk_mode = force_chunk_mode # Validate topts = IO.validate_raw_task_options(registered_tasks, topts) log.debug("Resolved task options to {d}".format(d=workflow_level_opts)) log.debug(pprint.pprint(workflow_level_opts.to_dict(), indent=4)) if isinstance(workflow_level_opts.cluster_manager_path, str): cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path) # override distributed mode if isinstance(force_distribute, bool): workflow_level_opts.distributed_mode = force_distribute else: cluster_render = None workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS) if workflow_level_opts.distributed_mode is False: slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count())) workflow_level_opts.total_max_nproc = multiprocessing.cpu_count() if debug_mode is True: slog.info("overriding debug-mode to True") workflow_level_opts.debug_mode = debug_mode return workflow_level_opts, topts, cluster_render
def validate_preset_xml(dir_name): from pbsmrtpipe.pb_io import parse_pipeline_preset_xml, validate_raw_task_options import pbsmrtpipe.loader as L rtasks_d, _, _, pts = L.load_all() for file_name in os.listdir(dir_name): if file_name.endswith(".xml"): p = parse_pipeline_preset_xml(op.join(dir_name, file_name)) if p.pipeline_id is None: raise ValueError("{f} does not have pipeline-id set".format( f=file_name)) elif not p.pipeline_id in pts: raise ValueError("pipeline-id {i} not recognized".format( i=p.pipeline_id)) log.info("validating {f}...".format(f=file_name)) validate_raw_task_options(rtasks_d, dict(p.task_options)) else: log.warn("Skipping non-XML file {f}".format(f=file_name)) return 0
def get_task_and_workflow_options(testkit_cfg): parsed_cfg = config_parser_to_butler(testkit_cfg) workflow_options, task_options = [], [] def __get_option_type(val): option_type = TaskOptionTypes.STR if isinstance(val, bool): option_type = TaskOptionTypes.BOOL elif isinstance(val, int): option_type = TaskOptionTypes.INT elif isinstance(val, float): option_type = TaskOptionTypes.FLOAT elif val is None: val = "" return option_type, val rtasks_d, _, _, _ = L.load_all() if not parsed_cfg.preset_xml in [None, '']: if not parsed_cfg.preset_json in [None, '']: raise ValueError( "Please use either preset_json or preset_xml, not both") presets = parse_pipeline_preset_xml(parsed_cfg.preset_xml) task_opts_d = validate_raw_task_options(rtasks_d, dict(presets.task_options)) for option_id, option_value in task_opts_d.iteritems(): log.info("task_option: {i} = {v}".format(i=option_id, v=option_value)) option_type, option_value = __get_option_type(option_value) task_options.append( dict(optionId=option_id, value=option_value, optionTypeId=option_type)) for option_id, option_value in presets.workflow_options: log.info("workflow_option: {i} = {v}".format(i=option_id, v=option_value)) workflow_options.append( dict(optionId=option_id, value=option_value, optionTypeId=__get_option_type(option_value)[0])) elif not parsed_cfg.preset_json in [None, '']: presets = parse_pipeline_preset_json(parsed_cfg.preset_json) for option_id, option_value in presets.task_options: log.info("task_option: {i} = {v}".format(i=option_id, v=option_value)) option_type, option_value = __get_option_type(option_value) task_options.append( dict(optionId=option_id, value=option_value, optionTypeId=option_type)) for option_id, option_value in presets.workflow_options: log.info("workflow_option: {i} = {v}".format(i=option_id, v=option_value)) workflow_options.append( dict(optionId=option_id, value=option_value, optionTypeId=__get_option_type(option_value)[0])) return task_options, workflow_options
def validate_preset_xml(dir_name): from pbsmrtpipe.pb_io import parse_pipeline_preset_xml, validate_raw_task_options import pbsmrtpipe.loader as L rtasks_d, _, _, pts = L.load_all() for file_name in os.listdir(dir_name): if file_name.endswith(".xml"): p = parse_pipeline_preset_xml(op.join(dir_name, file_name)) if p.pipeline_id is None: raise ValueError( "{f} does not have pipeline-id set".format(f=file_name)) elif not p.pipeline_id in pts: raise ValueError( "pipeline-id {i} not recognized".format(i=p.pipeline_id)) log.info("validating {f}...".format(f=file_name)) validate_raw_task_options(rtasks_d, dict(p.task_options)) else: log.warn("Skipping non-XML file {f}".format(f=file_name)) return 0
def _load_io_for_workflow(registered_tasks, registered_pipelines, workflow_template_xml_or_pipeline, entry_points_d, preset_xmls, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None): """ Load and resolve input IO layer # Load Presets and Workflow Options. Resolve and Merge # The Order of loading is # - rc, workflow.xml, then preset.xml # force_distribute will attempt to override ALL settings (if cluster_manager is defined) :returns: A tuple of Workflow Bindings, Workflow Level Options, Task Opts, ClusterRenderer) :rtype: (List[(str, str)], WorkflowLevelOpts, {TaskId:value}, ClusterRenderer) """ # Load Presets and Workflow Options. Resolve and Merge # The Order of loading is # - rc, workflow.xml, then preset.xml # A little sanity check # Validate that entry points exist slog.info("validating entry points.") _validate_entry_points_or_raise(entry_points_d) slog.info("successfully validated {n} entry points".format(n=len(entry_points_d))) wopts = {} topts = {} if rc_preset_or_none is None: rc_preset = IO.load_preset_from_env() else: rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none) if isinstance(workflow_template_xml_or_pipeline, Pipeline): # Use default values defined in the Pipeline builder_record = IO.BuilderRecord(workflow_template_xml_or_pipeline.all_bindings, workflow_template_xml_or_pipeline.task_options, {}) else: slog.info("Loading workflow template.") builder_record = IO.parse_pipeline_template_xml(workflow_template_xml_or_pipeline, registered_pipelines) slog.info("successfully loaded workflow template.") if preset_xmls: slog.info("Loading preset(s) {p}".format(p=preset_xmls)) preset_record = IO.parse_pipeline_preset_xmls(preset_xmls) slog.info("successfully loaded preset.") else: slog.info("No preset provided. Skipping preset.xml loading.") preset_record = None if rc_preset is not None: topts.update(dict(rc_preset.task_options)) wopts.update(dict(rc_preset.workflow_options)) wopts.update(dict(builder_record.workflow_options)) topts.update(builder_record.task_options) if preset_record is not None: wopts.update(dict(preset_record.workflow_options)) topts.update(dict(preset_record.task_options)) workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts) # override distributed mode only if provided. if isinstance(force_distribute, bool): workflow_level_opts.distributed_mode = force_distribute workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts) slog.info("Successfully validated workflow options.") slog.info("validating supplied task options.") topts = IO.validate_raw_task_options(registered_tasks, topts) slog.info("successfully validated (pre DI) task options.") workflow_bindings = builder_record.bindings if isinstance(workflow_level_opts.cluster_manager_path, str): cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path) else: cluster_render = None if isinstance(force_chunk_mode, bool): workflow_level_opts.chunk_mode = force_chunk_mode workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS) if workflow_level_opts.distributed_mode is False: slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count())) workflow_level_opts.total_max_nproc = multiprocessing.cpu_count() workflow_level_opts.max_nproc = multiprocessing.cpu_count() - 1 if debug_mode is True: slog.info("overriding debug-mode to True") workflow_level_opts.debug_mode = debug_mode log.debug("Resolved workflow level options to {d}".format(d=workflow_level_opts)) log.debug("\n" + pprint.pformat(workflow_level_opts.to_dict(), indent=4)) log.debug("Initial resolving of loaded preset.xml and pipeline.xml task options:") log.debug("\n" + pprint.pformat(topts)) return workflow_bindings, workflow_level_opts, topts, cluster_render
def _load_io_for_workflow(registered_tasks, registered_pipelines, workflow_template_xml_or_pipeline, entry_points_d, preset_xml, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None): """ Load and resolve input IO layer # Load Presets and Workflow Options. Resolve and Merge # The Order of loading is # - rc, workflow.xml, then preset.xml # force_distribute will attempt to override ALL settings (if cluster_manager is defined) :returns: A tuple of Workflow Bindings, Workflow Level Options, Task Opts, ClusterRenderer) :rtype: (List[(str, str)], WorkflowLevelOpts, {TaskId:value}, ClusterRenderer) """ # Load Presets and Workflow Options. Resolve and Merge # The Order of loading is # - rc, workflow.xml, then preset.xml # A little sanity check # Validate that entry points exist slog.info("validating entry points.") _validate_entry_points_or_raise(entry_points_d) slog.info("successfully validated {n} entry points".format(n=len(entry_points_d))) wopts = {} topts = {} if rc_preset_or_none is None: rc_preset = IO.load_preset_from_env() else: rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none) if isinstance(workflow_template_xml_or_pipeline, Pipeline): builder_record = IO.BuilderRecord(workflow_template_xml_or_pipeline.all_bindings, {}, {}) else: slog.info("Loading workflow template.") builder_record = IO.parse_pipeline_template_xml(workflow_template_xml_or_pipeline, registered_pipelines) slog.info("successfully loaded workflow template.") if preset_xml is None: slog.info("No preset provided. Skipping preset.xml loading.") preset_record = None else: slog.info("Loading preset {p}".format(p=preset_xml)) preset_record = IO.parse_pipeline_preset_xml(preset_xml) slog.info("successfully loaded preset.") if rc_preset is not None: topts.update(dict(rc_preset.task_options)) wopts.update(dict(rc_preset.workflow_options)) wopts.update(dict(builder_record.workflow_options)) topts.update(builder_record.task_options) if preset_record is not None: wopts.update(dict(preset_record.workflow_options)) topts.update(dict(preset_record.task_options)) workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts) # override distributed mode only if provided. if isinstance(force_distribute, bool): workflow_level_opts.distributed_mode = force_distribute workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts) slog.info("Successfully validated workflow options.") slog.info("validating supplied task options.") topts = IO.validate_raw_task_options(registered_tasks, topts) slog.info("successfully validated (pre DI) task options.") log.debug("Resolved task options to {d}".format(d=workflow_level_opts)) log.debug(pprint.pformat(workflow_level_opts.to_dict(), indent=4)) workflow_bindings = builder_record.bindings if isinstance(workflow_level_opts.cluster_manager_path, str): cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path) else: cluster_render = None if isinstance(force_chunk_mode, bool): workflow_level_opts.chunk_mode = force_chunk_mode workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS) if workflow_level_opts.distributed_mode is False: slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count())) workflow_level_opts.total_max_nproc = multiprocessing.cpu_count() if debug_mode is True: slog.info("overriding debug-mode to True") workflow_level_opts.debug_mode = debug_mode return workflow_bindings, workflow_level_opts, topts, cluster_render