def __init__(self, scatter_meta_task, original_nid, original_task_type_id, instance_id, chunk_group_id):
validate_type_or_raise(scatter_meta_task, (MetaScatterTask, ScatterToolContractMetaTask))
super(TaskScatterBindingNode, self).__init__(scatter_meta_task, instance_id)
# Keep track of the original task type that was chunked
self.original_task_id = original_task_type_id
self.original_nid = original_nid
self.chunk_group_id = chunk_group_id
def __init__(self, meta_task, instance_id, index, file_type_instance):
"""
:type meta_task: MetaTask
:type instance_id: int
:type index: int
:type file_type_instance: FileType
"""
# FileTypes are unique by (In/Out, file-type, instance-id)
# Trying to use int which are more friendly and make graph easier
# to view, but have to manually assigned in a centralized place
# This was not a great design choice
self.meta_task = validate_type_or_raise(meta_task, MetaTask)
# task index (int)
self.instance_id = validate_type_or_raise(instance_id, int)
# positional index of input/output
self.index = validate_type_or_raise(index, int)
# this is a little odd. The input/output type are not necessarily identical
self.file_klass = validate_type_or_raise(file_type_instance, FileType)
# this is the {file klass}-{Instance id}
types_ = getattr(self.meta_task, self.__class__.ATTR_NAME)
file_type = types_[self.index]
self.task_instance_id = "-".join(
[file_type.file_type_id, str(instance_id)])
def __init__(self, meta_task, instance_id, chunk_key):
validate_type_or_raise(meta_task,
(MetaGatherTask, GatherToolContractMetaTask))
super(TaskGatherBindingNode, self).__init__(meta_task, instance_id)
# Keep track of the chunk_key that was passed to the exe.
# Perhaps this should be in the meta task instance?
self.chunk_key = chunk_key
def __init__(self, scatter_meta_task, original_nid, original_task_type_id,
instance_id, chunk_group_id):
validate_type_or_raise(scatter_meta_task,
(MetaScatterTask, ScatterToolContractMetaTask))
super(TaskScatterBindingNode, self).__init__(scatter_meta_task,
instance_id)
# Keep track of the original task type that was chunked
self.original_task_id = original_task_type_id
self.original_nid = original_nid
self.chunk_group_id = chunk_group_id
def __init__(self, meta_task, instance_id):
"""
:type meta_task: MetaTask
:type instance_id: int
:param meta_task:
:param instance_id:
"""
self.meta_task = validate_type_or_raise(meta_task, (MetaTask, ToolContractMetaTask))
self.instance_id = validate_type_or_raise(instance_id, int)
def __init__(self, meta_task, instance_id):
"""
:type meta_task: MetaTask
:type instance_id: int
:param meta_task:
:param instance_id:
"""
self.meta_task = validate_type_or_raise(
meta_task, (MetaTask, ToolContractMetaTask))
self.instance_id = validate_type_or_raise(instance_id, int)
def __init__(self, meta_task, instance_id, index, file_type_instance):
"""
:type meta_task: MetaTask
:type instance_id: int
:type index: int
:type file_type_instance: FileType
"""
# FileTypes are unique by (In/Out, file-type, instance-id)
# Trying to use int which are more friendly and make graph easier
# to view, but have to manually assigned in a centralized place
# This was not a great design choice
self.meta_task = validate_type_or_raise(meta_task, MetaTask)
# task index (int)
self.instance_id = validate_type_or_raise(instance_id, int)
# positional index of input/output
self.index = validate_type_or_raise(index, int)
# this is a little odd. The input/output type are not necessarily identical
self.file_klass = validate_type_or_raise(file_type_instance, FileType)
def __init__(self, meta_task, instance_id, chunk_key):
validate_type_or_raise(meta_task, (MetaGatherTask, GatherToolContractMetaTask))
super(TaskGatherBindingNode, self).__init__(meta_task, instance_id)
# Keep track of the chunk_key that was passed to the exe.
# Perhaps this should be in the meta task instance?
self.chunk_key = chunk_key
def load_pipeline_bindings(registered_pipeline_d, pipeline_id, display_name, version, description, bs, tags, task_options):
"""
Mutate the registered pipelines registry
:param registered_pipeline_d:
:param pipeline_id:
:param bs: list of binding strings [(a, b), ]
:return: mutated pipeline registry
"""
# only use unique pairs
bs = list({x for x in bs})
log.debug("Processing pipeline {i}".format(i=pipeline_id))
# str, [(in, out)] [(in, out)]
pipeline = Pipeline(pipeline_id, display_name, version, description, [], [], tags=tags, task_options=task_options)
for x in bs:
validate_type_or_raise(x, (tuple, list))
if len(x) != 2:
raise TypeError("Binding Strings must be provided a 2-tuple of strings")
b_out, b_in = x
for x in (b_out, b_in):
is_validate_binding_str(x)
# Is it an Entry Point
if binding_str_is_entry_id(b_out):
# 3 cases, b_in is a
# - task_id
# - pipeline_id:entry_label (Rebound entry label)
# - pipeline_id:task_id (Using the output of an existing task in the pipeline)
# b_in could be a pipeline id or a task id
if binding_str_is_pipeline_task_str(b_in):
# print ("entry point -> pipeline", b_in)
# Need to load existing pipeline
# pipeline.entry_bindings.append((b_out, b_in))
# print "(load pipeline) entry points need to load existing pipeline for tasks and entry points", b_in
pass
elif binding_str_is_task_id(b_in):
# ($entry:e_01, "pbsmrtpipe.tasks.dev_task_01:0)
pipeline.entry_bindings.add((b_out, b_in))
elif _binding_str_match(RX_BINDING_PIPELINE_ENTRY, b_in):
# ($entry:e_01, pbsmrtpipe.pipelines.pipeline_id_1:$entry:e_02)
pi_id = get_pipeline_id_from_pipeline_entry_str(b_in)
e_label = get_entry_label_from_pipeline_entry_str(b_in)
_load_existing_pipeline_or_raise(registered_pipeline_d, pipeline, pi_id)
log.info("entry points -> pipeline:$entry format '{n}'".format(n=b_in))
log.debug("(re-bind) entry points need to load exiting pipeline for tasks and entry points")
else:
raise MalformedPipelineError("Unsupported value {b}".format(b=b_in))
# is regular task -> task bindings
elif binding_str_is_task_id(b_out):
# simplest case
# print ("task -> task binding", b_out, b_in)
pipeline.bindings.add((b_out, b_in))
elif _binding_str_match(RX_BINDING_PIPELINE_TASK, b_out):
# pbsmrtpipe.pipelines.dev_01:pbsmrtpipe.tasks.dev_hello_world:0
# needs to load existing pipeline bindings and entry points
# then create a new binding of ("pbsmrtpipe.tasks.dev_hello_world:0", b_in)
task_binding_str = get_task_binding_str_from_pipeline_task_str(b_out)
pl_id = get_pipeline_id_from_pipeline_task_str(b_out)
_load_existing_pipeline_or_raise(registered_pipeline_d, pipeline, pl_id)
pipeline.bindings.add((task_binding_str, b_in))
# print ("pipeline task binding", b_out, b_in)
else:
raise MalformedPipelineError("Unhandled binding case '{o}' -> '{i}'".format(o=b_out, i=b_in))
if pipeline.pipeline_id not in registered_pipeline_d:
log.debug("registering pipeline {i}".format(i=pipeline.pipeline_id))
registered_pipeline_d[pipeline.pipeline_id] = pipeline
return registered_pipeline_d
def load_pipeline_bindings(registered_pipeline_d, pipeline_id, display_name, version, description, bs, tags):
"""
Mutate the registered pipelines registry
:param registered_pipeline_d:
:param pipeline_id:
:param bs: list of binding strings [(a, b), ]
:return: mutated pipeline registry
"""
# only use unique pairs
bs = list({x for x in bs})
log.debug("Processing pipeline {i}".format(i=pipeline_id))
# str, [(in, out)] [(in, out)]
pipeline = Pipeline(pipeline_id, display_name, version, description, [], [], tags=tags)
for x in bs:
validate_type_or_raise(x, (tuple, list))
if len(x) != 2:
raise TypeError("Binding Strings must be provided a 2-tuple of strings")
b_out, b_in = x
for x in (b_out, b_in):
is_validate_binding_str(x)
# Is it an Entry Point
if binding_str_is_entry_id(b_out):
# 3 cases, b_in is a
# - task_id
# - pipeline_id:entry_label (Rebound entry label)
# - pipeline_id:task_id (Using the output of an existing task in the pipeline)
# b_in could be a pipeline id or a task id
if binding_str_is_pipeline_task_str(b_in):
# print ("entry point -> pipeline", b_in)
# Need to load existing pipeline
# pipeline.entry_bindings.append((b_out, b_in))
# print "(load pipeline) entry points need to load existing pipeline for tasks and entry points", b_in
pass
elif binding_str_is_task_id(b_in):
# ($entry:e_01, "pbsmrtpipe.tasks.dev_task_01:0)
pipeline.entry_bindings.append((b_out, b_in))
elif _binding_str_match(RX_BINDING_PIPELINE_ENTRY, b_in):
# ($entry:e_01, pbsmrtpipe.pipelines.pipeline_id_1:$entry:e_02)
pi_id = get_pipeline_id_from_pipeline_entry_str(b_in)
e_label = get_entry_label_from_pipeline_entry_str(b_in)
_load_existing_pipeline_or_raise(registered_pipeline_d, pipeline, pi_id)
log.info("entry points -> pipeline:$entry format '{n}'".format(n=b_in))
log.debug("(re-bind) entry points need to load exiting pipeline for tasks and entry points")
else:
raise MalformedPipelineError("Unsupported value {b}".format(b=b_in))
# is regular task -> task bindings
elif binding_str_is_task_id(b_out):
# simplest case
# print ("task -> task binding", b_out, b_in)
pipeline.bindings.append((b_out, b_in))
elif _binding_str_match(RX_BINDING_PIPELINE_TASK, b_out):
# pbsmrtpipe.pipelines.dev_01:pbsmrtpipe.tasks.dev_hello_world:0
# needs to load existing pipeline bindings and entry points
# then create a new binding of ("pbsmrtpipe.tasks.dev_hello_world:0", b_in)
task_binding_str = get_task_binding_str_from_pipeline_task_str(b_out)
pl_id = get_pipeline_id_from_pipeline_task_str(b_out)
_load_existing_pipeline_or_raise(registered_pipeline_d, pipeline, pl_id)
pipeline.bindings.append((task_binding_str, b_in))
# print ("pipeline task binding", b_out, b_in)
else:
raise MalformedPipelineError("Unhandled binding case '{o}' -> '{i}'".format(o=b_out, i=b_in))
log.info("registering pipeline {i}".format(i=pipeline.pipeline_id))
registered_pipeline_d[pipeline.pipeline_id] = pipeline
return registered_pipeline_d
