def load_pipeline_template_from(d_or_path):
"""
Load a Resolved Pipeline Template from a JSON file
:type path: str
:rtype: Pipeline
"""
d = _load_from_dict_or_path(d_or_path)
bindings = {_load_bindings(x) for x in d['bindings']}
epoints = list(
itertools.chain(*[_load_entry_binding(ei) for ei in d['entryPoints']]))
pacbio_task_options = [
pacbio_option_from_dict(opt_d) for opt_d in d['taskOptions']
]
# The pipeline instance only needs to the key-value pair
task_options = {
pbopt.option_id: pbopt.default
for pbopt in pacbio_task_options
}
p = Pipeline(d['id'],
d['name'],
d['version'],
d['description'],
bindings,
epoints,
tags=d['tags'],
task_options=task_options)
return p
def load_pipeline_template_from(d_or_path):
"""
Load a Resolved Pipeline Template from a JSON file
:type path: str
:rtype: Pipeline
"""
d = _load_from_dict_or_path(d_or_path)
bindings = {_load_bindings(x) for x in d['bindings']}
epoints = list(
itertools.chain(*[_load_entry_binding(ei) for ei in d['entryPoints']]))
p = Pipeline(d['id'],
d['name'],
d['version'],
d['description'],
bindings,
epoints,
tags=d['tags'])
return p
def load_pipeline_bindings(registered_pipeline_d, pipeline_id, display_name, version, description, bs, tags):
"""
Mutate the registered pipelines registry
:param registered_pipeline_d:
:param pipeline_id:
:param bs: list of binding strings [(a, b), ]
:return: mutated pipeline registry
"""
# only use unique pairs
bs = list({x for x in bs})
log.debug("Processing pipeline {i}".format(i=pipeline_id))
# str, [(in, out)] [(in, out)]
pipeline = Pipeline(pipeline_id, display_name, version, description, [], [], tags=tags)
for x in bs:
validate_type_or_raise(x, (tuple, list))
if len(x) != 2:
raise TypeError("Binding Strings must be provided a 2-tuple of strings")
b_out, b_in = x
for x in (b_out, b_in):
is_validate_binding_str(x)
# Is it an Entry Point
if binding_str_is_entry_id(b_out):
# 3 cases, b_in is a
# - task_id
# - pipeline_id:entry_label (Rebound entry label)
# - pipeline_id:task_id (Using the output of an existing task in the pipeline)
# b_in could be a pipeline id or a task id
if binding_str_is_pipeline_task_str(b_in):
# print ("entry point -> pipeline", b_in)
# Need to load existing pipeline
# pipeline.entry_bindings.append((b_out, b_in))
# print "(load pipeline) entry points need to load existing pipeline for tasks and entry points", b_in
pass
elif binding_str_is_task_id(b_in):
# ($entry:e_01, "pbsmrtpipe.tasks.dev_task_01:0)
pipeline.entry_bindings.append((b_out, b_in))
elif _binding_str_match(RX_BINDING_PIPELINE_ENTRY, b_in):
# ($entry:e_01, pbsmrtpipe.pipelines.pipeline_id_1:$entry:e_02)
pi_id = get_pipeline_id_from_pipeline_entry_str(b_in)
e_label = get_entry_label_from_pipeline_entry_str(b_in)
_load_existing_pipeline_or_raise(registered_pipeline_d, pipeline, pi_id)
log.info("entry points -> pipeline:$entry format '{n}'".format(n=b_in))
log.debug("(re-bind) entry points need to load exiting pipeline for tasks and entry points")
else:
raise MalformedPipelineError("Unsupported value {b}".format(b=b_in))
# is regular task -> task bindings
elif binding_str_is_task_id(b_out):
# simplest case
# print ("task -> task binding", b_out, b_in)
pipeline.bindings.append((b_out, b_in))
elif _binding_str_match(RX_BINDING_PIPELINE_TASK, b_out):
# pbsmrtpipe.pipelines.dev_01:pbsmrtpipe.tasks.dev_hello_world:0
# needs to load existing pipeline bindings and entry points
# then create a new binding of ("pbsmrtpipe.tasks.dev_hello_world:0", b_in)
task_binding_str = get_task_binding_str_from_pipeline_task_str(b_out)
pl_id = get_pipeline_id_from_pipeline_task_str(b_out)
_load_existing_pipeline_or_raise(registered_pipeline_d, pipeline, pl_id)
pipeline.bindings.append((task_binding_str, b_in))
# print ("pipeline task binding", b_out, b_in)
else:
raise MalformedPipelineError("Unhandled binding case '{o}' -> '{i}'".format(o=b_out, i=b_in))
log.info("registering pipeline {i}".format(i=pipeline.pipeline_id))
registered_pipeline_d[pipeline.pipeline_id] = pipeline
return registered_pipeline_d