def _load_io_for_task(registered_tasks, entry_points_d, preset_xml, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None):
"""Grungy loading of the IO and resolving values
Returns a tuple of (WorkflowLevelOptions, TaskOptions, ClusterRender)
"""
slog.info("validating entry points. {e}".format(e=entry_points_d))
_validate_entry_points_or_raise(entry_points_d)
slog.info("successfully validated {n} entry points".format(n=len(entry_points_d)))
wopts = {}
topts = {}
if rc_preset_or_none is None:
rc_preset = IO.load_preset_from_env()
else:
rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none)
if rc_preset:
topts.update(dict(rc_preset.task_options))
wopts.update(dict(rc_preset.workflow_options))
if preset_xml is not None:
preset_record = IO.parse_pipeline_preset_xml(preset_xml)
wopts.update(dict(preset_record.workflow_options))
topts.update(dict(preset_record.task_options))
workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts)
workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts)
if isinstance(force_chunk_mode, bool):
workflow_level_opts.chunk_mode = force_chunk_mode
# Validate
topts = IO.validate_raw_task_options(registered_tasks, topts)
log.debug("Resolved task options to {d}".format(d=workflow_level_opts))
log.debug(pprint.pprint(workflow_level_opts.to_dict(), indent=4))
if isinstance(workflow_level_opts.cluster_manager_path, str):
cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path)
# override distributed mode
if isinstance(force_distribute, bool):
workflow_level_opts.distributed_mode = force_distribute
else:
cluster_render = None
workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS)
if workflow_level_opts.distributed_mode is False:
slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count()))
workflow_level_opts.total_max_nproc = multiprocessing.cpu_count()
if debug_mode is True:
slog.info("overriding debug-mode to True")
workflow_level_opts.debug_mode = debug_mode
return workflow_level_opts, topts, cluster_render
def _load_io_for_task(registered_tasks, entry_points_d, preset_xmls, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None):
"""Grungy loading of the IO and resolving values
Returns a tuple of (WorkflowLevelOptions, TaskOptions, ClusterRender)
"""
slog.info("validating entry points. {e}".format(e=entry_points_d))
_validate_entry_points_or_raise(entry_points_d)
slog.info("successfully validated {n} entry points".format(n=len(entry_points_d)))
wopts = {}
topts = {}
if rc_preset_or_none is None:
rc_preset = IO.load_preset_from_env()
else:
rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none)
if rc_preset:
topts.update(dict(rc_preset.task_options))
wopts.update(dict(rc_preset.workflow_options))
if preset_xmls:
preset_record = IO.parse_pipeline_preset_xmls(preset_xmls)
wopts.update(dict(preset_record.workflow_options))
topts.update(dict(preset_record.task_options))
workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts)
workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts)
if isinstance(force_chunk_mode, bool):
workflow_level_opts.chunk_mode = force_chunk_mode
# Validate
topts = IO.validate_raw_task_options(registered_tasks, topts)
log.debug("Resolved task options to {d}".format(d=workflow_level_opts))
log.debug(pprint.pprint(workflow_level_opts.to_dict(), indent=4))
if isinstance(workflow_level_opts.cluster_manager_path, str):
cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path)
# override distributed mode
if isinstance(force_distribute, bool):
workflow_level_opts.distributed_mode = force_distribute
else:
cluster_render = None
workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS)
if workflow_level_opts.distributed_mode is False:
slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count()))
workflow_level_opts.total_max_nproc = multiprocessing.cpu_count()
if debug_mode is True:
slog.info("overriding debug-mode to True")
workflow_level_opts.debug_mode = debug_mode
return workflow_level_opts, topts, cluster_render
def validate_preset_xml(dir_name):
from pbsmrtpipe.pb_io import parse_pipeline_preset_xml, validate_raw_task_options
import pbsmrtpipe.loader as L
rtasks_d, _, _, pts = L.load_all()
for file_name in os.listdir(dir_name):
if file_name.endswith(".xml"):
p = parse_pipeline_preset_xml(op.join(dir_name, file_name))
if p.pipeline_id is None:
raise ValueError("{f} does not have pipeline-id set".format(
f=file_name))
elif not p.pipeline_id in pts:
raise ValueError("pipeline-id {i} not recognized".format(
i=p.pipeline_id))
log.info("validating {f}...".format(f=file_name))
validate_raw_task_options(rtasks_d, dict(p.task_options))
else:
log.warn("Skipping non-XML file {f}".format(f=file_name))
return 0
def get_task_and_workflow_options(testkit_cfg):
parsed_cfg = config_parser_to_butler(testkit_cfg)
workflow_options, task_options = [], []
def __get_option_type(val):
option_type = TaskOptionTypes.STR
if isinstance(val, bool):
option_type = TaskOptionTypes.BOOL
elif isinstance(val, int):
option_type = TaskOptionTypes.INT
elif isinstance(val, float):
option_type = TaskOptionTypes.FLOAT
elif val is None:
val = ""
return option_type, val
rtasks_d, _, _, _ = L.load_all()
if not parsed_cfg.preset_xml in [None, '']:
if not parsed_cfg.preset_json in [None, '']:
raise ValueError(
"Please use either preset_json or preset_xml, not both")
presets = parse_pipeline_preset_xml(parsed_cfg.preset_xml)
task_opts_d = validate_raw_task_options(rtasks_d,
dict(presets.task_options))
for option_id, option_value in task_opts_d.iteritems():
log.info("task_option: {i} = {v}".format(i=option_id,
v=option_value))
option_type, option_value = __get_option_type(option_value)
task_options.append(
dict(optionId=option_id,
value=option_value,
optionTypeId=option_type))
for option_id, option_value in presets.workflow_options:
log.info("workflow_option: {i} = {v}".format(i=option_id,
v=option_value))
workflow_options.append(
dict(optionId=option_id,
value=option_value,
optionTypeId=__get_option_type(option_value)[0]))
elif not parsed_cfg.preset_json in [None, '']:
presets = parse_pipeline_preset_json(parsed_cfg.preset_json)
for option_id, option_value in presets.task_options:
log.info("task_option: {i} = {v}".format(i=option_id,
v=option_value))
option_type, option_value = __get_option_type(option_value)
task_options.append(
dict(optionId=option_id,
value=option_value,
optionTypeId=option_type))
for option_id, option_value in presets.workflow_options:
log.info("workflow_option: {i} = {v}".format(i=option_id,
v=option_value))
workflow_options.append(
dict(optionId=option_id,
value=option_value,
optionTypeId=__get_option_type(option_value)[0]))
return task_options, workflow_options
def validate_preset_xml(dir_name):
from pbsmrtpipe.pb_io import parse_pipeline_preset_xml, validate_raw_task_options
import pbsmrtpipe.loader as L
rtasks_d, _, _, pts = L.load_all()
for file_name in os.listdir(dir_name):
if file_name.endswith(".xml"):
p = parse_pipeline_preset_xml(op.join(dir_name, file_name))
if p.pipeline_id is None:
raise ValueError(
"{f} does not have pipeline-id set".format(f=file_name))
elif not p.pipeline_id in pts:
raise ValueError(
"pipeline-id {i} not recognized".format(i=p.pipeline_id))
log.info("validating {f}...".format(f=file_name))
validate_raw_task_options(rtasks_d, dict(p.task_options))
else:
log.warn("Skipping non-XML file {f}".format(f=file_name))
return 0
def _load_io_for_workflow(registered_tasks, registered_pipelines, workflow_template_xml_or_pipeline,
entry_points_d, preset_xmls, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None):
"""
Load and resolve input IO layer
# Load Presets and Workflow Options. Resolve and Merge
# The Order of loading is
# - rc, workflow.xml, then preset.xml
# force_distribute will attempt to override ALL settings (if cluster_manager is defined)
:returns: A tuple of Workflow Bindings, Workflow Level Options, Task Opts, ClusterRenderer)
:rtype: (List[(str, str)], WorkflowLevelOpts, {TaskId:value}, ClusterRenderer)
"""
# Load Presets and Workflow Options. Resolve and Merge
# The Order of loading is
# - rc, workflow.xml, then preset.xml
# A little sanity check
# Validate that entry points exist
slog.info("validating entry points.")
_validate_entry_points_or_raise(entry_points_d)
slog.info("successfully validated {n} entry points".format(n=len(entry_points_d)))
wopts = {}
topts = {}
if rc_preset_or_none is None:
rc_preset = IO.load_preset_from_env()
else:
rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none)
if isinstance(workflow_template_xml_or_pipeline, Pipeline):
# Use default values defined in the Pipeline
builder_record = IO.BuilderRecord(workflow_template_xml_or_pipeline.all_bindings, workflow_template_xml_or_pipeline.task_options, {})
else:
slog.info("Loading workflow template.")
builder_record = IO.parse_pipeline_template_xml(workflow_template_xml_or_pipeline, registered_pipelines)
slog.info("successfully loaded workflow template.")
if preset_xmls:
slog.info("Loading preset(s) {p}".format(p=preset_xmls))
preset_record = IO.parse_pipeline_preset_xmls(preset_xmls)
slog.info("successfully loaded preset.")
else:
slog.info("No preset provided. Skipping preset.xml loading.")
preset_record = None
if rc_preset is not None:
topts.update(dict(rc_preset.task_options))
wopts.update(dict(rc_preset.workflow_options))
wopts.update(dict(builder_record.workflow_options))
topts.update(builder_record.task_options)
if preset_record is not None:
wopts.update(dict(preset_record.workflow_options))
topts.update(dict(preset_record.task_options))
workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts)
# override distributed mode only if provided.
if isinstance(force_distribute, bool):
workflow_level_opts.distributed_mode = force_distribute
workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts)
slog.info("Successfully validated workflow options.")
slog.info("validating supplied task options.")
topts = IO.validate_raw_task_options(registered_tasks, topts)
slog.info("successfully validated (pre DI) task options.")
workflow_bindings = builder_record.bindings
if isinstance(workflow_level_opts.cluster_manager_path, str):
cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path)
else:
cluster_render = None
if isinstance(force_chunk_mode, bool):
workflow_level_opts.chunk_mode = force_chunk_mode
workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS)
if workflow_level_opts.distributed_mode is False:
slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count()))
workflow_level_opts.total_max_nproc = multiprocessing.cpu_count()
workflow_level_opts.max_nproc = multiprocessing.cpu_count() - 1
if debug_mode is True:
slog.info("overriding debug-mode to True")
workflow_level_opts.debug_mode = debug_mode
log.debug("Resolved workflow level options to {d}".format(d=workflow_level_opts))
log.debug("\n" + pprint.pformat(workflow_level_opts.to_dict(), indent=4))
log.debug("Initial resolving of loaded preset.xml and pipeline.xml task options:")
log.debug("\n" + pprint.pformat(topts))
return workflow_bindings, workflow_level_opts, topts, cluster_render
def _load_io_for_workflow(registered_tasks, registered_pipelines, workflow_template_xml_or_pipeline,
entry_points_d, preset_xml, rc_preset_or_none, force_distribute=None, force_chunk_mode=None, debug_mode=None):
"""
Load and resolve input IO layer
# Load Presets and Workflow Options. Resolve and Merge
# The Order of loading is
# - rc, workflow.xml, then preset.xml
# force_distribute will attempt to override ALL settings (if cluster_manager is defined)
:returns: A tuple of Workflow Bindings, Workflow Level Options, Task Opts, ClusterRenderer)
:rtype: (List[(str, str)], WorkflowLevelOpts, {TaskId:value}, ClusterRenderer)
"""
# Load Presets and Workflow Options. Resolve and Merge
# The Order of loading is
# - rc, workflow.xml, then preset.xml
# A little sanity check
# Validate that entry points exist
slog.info("validating entry points.")
_validate_entry_points_or_raise(entry_points_d)
slog.info("successfully validated {n} entry points".format(n=len(entry_points_d)))
wopts = {}
topts = {}
if rc_preset_or_none is None:
rc_preset = IO.load_preset_from_env()
else:
rc_preset = IO.parse_pipeline_preset_xml(rc_preset_or_none)
if isinstance(workflow_template_xml_or_pipeline, Pipeline):
builder_record = IO.BuilderRecord(workflow_template_xml_or_pipeline.all_bindings, {}, {})
else:
slog.info("Loading workflow template.")
builder_record = IO.parse_pipeline_template_xml(workflow_template_xml_or_pipeline, registered_pipelines)
slog.info("successfully loaded workflow template.")
if preset_xml is None:
slog.info("No preset provided. Skipping preset.xml loading.")
preset_record = None
else:
slog.info("Loading preset {p}".format(p=preset_xml))
preset_record = IO.parse_pipeline_preset_xml(preset_xml)
slog.info("successfully loaded preset.")
if rc_preset is not None:
topts.update(dict(rc_preset.task_options))
wopts.update(dict(rc_preset.workflow_options))
wopts.update(dict(builder_record.workflow_options))
topts.update(builder_record.task_options)
if preset_record is not None:
wopts.update(dict(preset_record.workflow_options))
topts.update(dict(preset_record.task_options))
workflow_level_opts = IO.WorkflowLevelOptions.from_id_dict(wopts)
# override distributed mode only if provided.
if isinstance(force_distribute, bool):
workflow_level_opts.distributed_mode = force_distribute
workflow_level_opts = IO.validate_or_modify_workflow_level_options(workflow_level_opts)
slog.info("Successfully validated workflow options.")
slog.info("validating supplied task options.")
topts = IO.validate_raw_task_options(registered_tasks, topts)
slog.info("successfully validated (pre DI) task options.")
log.debug("Resolved task options to {d}".format(d=workflow_level_opts))
log.debug(pprint.pformat(workflow_level_opts.to_dict(), indent=4))
workflow_bindings = builder_record.bindings
if isinstance(workflow_level_opts.cluster_manager_path, str):
cluster_render = C.load_cluster_templates(workflow_level_opts.cluster_manager_path)
else:
cluster_render = None
if isinstance(force_chunk_mode, bool):
workflow_level_opts.chunk_mode = force_chunk_mode
workflow_level_opts.max_nchunks = min(workflow_level_opts.max_nchunks, GlobalConstants.MAX_NCHUNKS)
if workflow_level_opts.distributed_mode is False:
slog.info("local-only mode detected setting total NPROC to {x}".format(x=multiprocessing.cpu_count()))
workflow_level_opts.total_max_nproc = multiprocessing.cpu_count()
if debug_mode is True:
slog.info("overriding debug-mode to True")
workflow_level_opts.debug_mode = debug_mode
return workflow_bindings, workflow_level_opts, topts, cluster_render
