def run_pipeline(registered_pipelines_d, registered_file_types_d, registered_tasks_d,
chunk_operators, workflow_template_xml, entry_points_d,
output_dir, preset_xmls, rc_preset_or_none, service_uri,
force_distribute=None, force_chunk_mode=None, debug_mode=None):
"""
Entry point for running a pipeline
:param workflow_template_xml: path to workflow xml
:param entry_points_d:
:param output_dir:
:param preset_xmls: list of path to preset xml
:return: exit code
:type registered_tasks_d: dict[str, pbsmrtpipe.pb_tasks.core.MetaTask]
:type registered_file_types_d: dict[str, pbsmrtpipe.pb_tasks.core.FileType]
:type workflow_template_xml: str
:type output_dir: str
:type preset_xmls: list[str]
:type service_uri: str | None
:type force_distribute: None | bool
:rtype: int
"""
log.debug(pprint.pformat(entry_points_d))
workflow_bindings, workflow_level_opts, task_opts, cluster_render = _load_io_for_workflow(registered_tasks_d,
registered_pipelines_d,
workflow_template_xml,
entry_points_d, preset_xmls,
rc_preset_or_none,
force_distribute=force_distribute,
force_chunk_mode=force_chunk_mode,
debug_mode=debug_mode)
slog.info("building graph")
bg = B.binding_strs_to_binding_graph(registered_tasks_d, workflow_bindings)
slog.info("successfully loaded graph from bindings.")
valid_chunk_operators = {}
# Disabled chunk operators if necessary
if workflow_level_opts.chunk_mode is False:
slog.info("Chunk mode is False. Disabling {n} chunk operators.".format(n=len(chunk_operators)))
else:
# Validate chunk operators, or skip if malformed.
for chunk_operator_id, chunk_operator in chunk_operators.iteritems():
try:
validate_operator(chunk_operator, registered_tasks_d)
valid_chunk_operators[chunk_operator_id] = chunk_operator
except MalformedChunkOperatorError as e:
log.warn("Invalid chunk operator {i}. {m}".format(i=chunk_operator_id, m=e.message))
filtered_chunk_operators_d = _filter_chunk_operators(bg, valid_chunk_operators)
# Container to hold all the resources
global_registry = GlobalRegistry(registered_tasks_d,
registered_file_types_d,
filtered_chunk_operators_d,
cluster_render)
return exe_workflow(global_registry, entry_points_d, bg, task_opts,
workflow_level_opts, output_dir, service_uri)
def run_pipeline(registered_pipelines_d, registered_file_types_d, registered_tasks_d,
chunk_operators, workflow_template_xml, entry_points_d,
output_dir, preset_xml, rc_preset_or_none, service_uri,
force_distribute=None, force_chunk_mode=None, debug_mode=None):
"""
Entry point for running a pipeline
:param workflow_template_xml: path to workflow xml
:param entry_points_d:
:param output_dir:
:param preset_xml: path to preset xml (or None)
:return: exit code
:type registered_tasks_d: dict[str, pbsmrtpipe.pb_tasks.core.MetaTask]
:type registered_file_types_d: dict[str, pbsmrtpipe.pb_tasks.core.FileType]
:type workflow_template_xml: str
:type output_dir: str
:type preset_xml: str | None
:type service_uri: str | None
:type force_distribute: None | bool
:rtype: int
"""
log.debug(pprint.pformat(entry_points_d))
workflow_bindings, workflow_level_opts, task_opts, cluster_render = _load_io_for_workflow(registered_tasks_d,
registered_pipelines_d,
workflow_template_xml,
entry_points_d, preset_xml,
rc_preset_or_none,
force_distribute=force_distribute,
force_chunk_mode=force_chunk_mode,
debug_mode=debug_mode)
slog.info("building graph")
bg = B.binding_strs_to_binding_graph(registered_tasks_d, workflow_bindings)
slog.info("successfully loaded graph from bindings.")
valid_chunk_operators = {}
# Disabled chunk operators if necessary
if workflow_level_opts.chunk_mode is False:
slog.info("Chunk mode is False. Disabling {n} chunk operators.".format(n=len(chunk_operators)))
else:
# Validate chunk operators, or skip if malformed.
for chunk_operator_id, chunk_operator in chunk_operators.iteritems():
try:
validate_operator(chunk_operator, registered_tasks_d)
valid_chunk_operators[chunk_operator_id] = chunk_operator
except MalformedChunkOperatorError as e:
log.warn("Invalid chunk operator {i}. {m}".format(i=chunk_operator_id, m=e.message))
filtered_chunk_operators_d = _filter_chunk_operators(bg, valid_chunk_operators)
# Container to hold all the resources
global_registry = GlobalRegistry(registered_tasks_d,
registered_file_types_d,
filtered_chunk_operators_d,
cluster_render)
return exe_workflow(global_registry, entry_points_d, bg, task_opts,
workflow_level_opts, output_dir, service_uri)
def run_single_task(registered_file_types_d,
registered_tasks_d,
chunk_operators,
entry_points_d,
task_id,
output_dir,
preset_xmls,
rc_preset_or_none,
service_config,
force_distribute=None,
force_chunk_mode=None,
debug_mode=None):
"""
Entry Point for running a single task
:param task_id:
:param output_dir:
:return:
"""
print entry_points_d
meta_task = registered_tasks_d.get(task_id, None)
if meta_task is None:
raise KeyError(
"Unable to find task id '{i}' in registered tasks. Use "
"'show-tasks' to get a list of registered tasks.".format(
i=task_id))
workflow_level_opts, task_opts, cluster_render = _load_io_for_task(
registered_tasks_d,
entry_points_d,
preset_xmls,
rc_preset_or_none,
force_distribute=force_distribute,
force_chunk_mode=force_chunk_mode,
debug_mode=debug_mode)
slog.info("building bindings graph")
binding_str = _task_to_binding_strings(meta_task)
bg = B.binding_strs_to_binding_graph(registered_tasks_d, binding_str)
slog.info("successfully bindings graph for task {i}".format(i=task_id))
# Validate chunk operators
valid_chunk_operators = {
k: v
for k, v in chunk_operators.iteritems()
if validate_operator(v, registered_tasks_d)
}
filtered_chunk_operators_d = _filter_chunk_operators(
bg, valid_chunk_operators)
# Container to hold all the resources
global_registry = GlobalRegistry(registered_tasks_d,
registered_file_types_d,
filtered_chunk_operators_d,
cluster_render)
return exe_workflow(global_registry, entry_points_d, bg, task_opts,
workflow_level_opts, output_dir, service_config)
def run_single_task(registered_file_types_d, registered_tasks_d, chunk_operators,
entry_points_d, task_id, output_dir, preset_xmls, rc_preset_or_none,
service_config,
force_distribute=None,
force_chunk_mode=None,
debug_mode=None):
"""
Entry Point for running a single task
:param task_id:
:param output_dir:
:return:
"""
print entry_points_d
meta_task = registered_tasks_d.get(task_id, None)
if meta_task is None:
raise KeyError("Unable to find task id '{i}' in registered tasks. Use "
"'show-tasks' to get a list of registered tasks.".format(i=task_id))
workflow_level_opts, task_opts, cluster_render = _load_io_for_task(registered_tasks_d, entry_points_d,
preset_xmls, rc_preset_or_none,
force_distribute=force_distribute,
force_chunk_mode=force_chunk_mode,
debug_mode=debug_mode)
slog.info("building bindings graph")
binding_str = _task_to_binding_strings(meta_task)
bg = B.binding_strs_to_binding_graph(registered_tasks_d, binding_str)
slog.info("successfully bindings graph for task {i}".format(i=task_id))
# Validate chunk operators
valid_chunk_operators = {k: v for k, v in chunk_operators.iteritems() if validate_operator(v, registered_tasks_d)}
filtered_chunk_operators_d = _filter_chunk_operators(bg, valid_chunk_operators)
# Container to hold all the resources
global_registry = GlobalRegistry(registered_tasks_d,
registered_file_types_d,
filtered_chunk_operators_d,
cluster_render)
return exe_workflow(global_registry, entry_points_d, bg, task_opts,
workflow_level_opts, output_dir, service_config)
