def replacement_for_connection(self, connection, is_data=True):
output_step_id = connection.output_step.id
if output_step_id not in self.outputs:
template = "No outputs found for step id %s, outputs are %s"
message = template % (output_step_id, self.outputs)
raise Exception(message)
step_outputs = self.outputs[output_step_id]
if step_outputs is STEP_OUTPUT_DELAYED:
delayed_why = "dependent step [%s] delayed, so this step must be delayed" % output_step_id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
output_name = connection.output_name
try:
replacement = step_outputs[output_name]
except KeyError:
if is_data:
# Must resolve.
template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s"
message = template % (output_name, step_outputs)
raise Exception(message)
else:
replacement = modules.NO_REPLACEMENT
if isinstance(replacement, model.HistoryDatasetCollectionAssociation):
if not replacement.collection.populated:
if not replacement.collection.waiting_for_elements:
# If we are not waiting for elements, there was some
# problem creating the collection. Collection will never
# be populated.
# TODO: consider distinguish between cancelled and failed?
raise modules.CancelWorkflowEvaluation()
delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
return replacement
def __check_implicitly_dependent_step(self, output_id):
step_invocation = self.workflow_invocation.step_invocation_for_step_id(
output_id)
# No steps created yet - have to delay evaluation.
if not step_invocation:
delayed_why = "depends on step [%s] but that step has not been invoked yet" % output_id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
if step_invocation.state != 'scheduled':
delayed_why = "depends on step [%s] job has not finished scheduling yet" % output_id
raise modules.DelayedWorkflowEvaluation(delayed_why)
for job_assoc in step_invocation.jobs:
job = job_assoc.job
if job:
# At least one job in incomplete.
if not job.finished:
delayed_why = "depends on step [%s] but one or more jobs created from that step have not finished yet" % output_id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
if job.state != job.states.OK:
raise modules.CancelWorkflowEvaluation()
else:
# TODO: Handle implicit dependency on stuff like
# pause steps.
pass
def replacement_for_connection(self, connection, is_data=True):
output_step_id = connection.output_step.id
if output_step_id not in self.outputs:
template = "No outputs found for step id %s, outputs are %s"
message = template % (output_step_id, self.outputs)
raise Exception(message)
step_outputs = self.outputs[output_step_id]
if step_outputs is STEP_OUTPUT_DELAYED:
delayed_why = "dependent step [%s] delayed, so this step must be delayed" % output_step_id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
output_name = connection.output_name
try:
replacement = step_outputs[output_name]
except KeyError:
replacement = self.inputs_by_step_id.get(output_step_id)
if connection.output_step.type == 'parameter_input' and output_step_id is not None:
# FIXME: parameter_input step outputs should be properly recorded as step outputs, but for now we can
# short-circuit and just pick the input value
pass
else:
# Must resolve.
template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s"
message = template % (output_name, step_outputs)
raise Exception(message)
if isinstance(replacement, model.HistoryDatasetCollectionAssociation):
if not replacement.collection.populated:
if not replacement.collection.waiting_for_elements:
# If we are not waiting for elements, there was some
# problem creating the collection. Collection will never
# be populated.
# TODO: consider distinguish between cancelled and failed?
raise modules.CancelWorkflowEvaluation()
delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
data_inputs = (model.HistoryDatasetAssociation,
model.HistoryDatasetCollectionAssociation,
model.DatasetCollection)
if not is_data and isinstance(replacement, data_inputs):
if isinstance(replacement, model.HistoryDatasetAssociation):
if replacement.is_pending:
raise modules.DelayedWorkflowEvaluation()
if not replacement.is_ok:
raise modules.CancelWorkflowEvaluation()
else:
if not replacement.collection.populated:
raise modules.DelayedWorkflowEvaluation()
pending = False
for dataset_instance in replacement.dataset_instances:
if dataset_instance.is_pending:
pending = True
elif not dataset_instance.is_ok:
raise modules.CancelWorkflowEvaluation()
if pending:
raise modules.DelayedWorkflowEvaluation()
return replacement
def replacement_for_connection(self, connection, is_data=True):
output_step_id = connection.output_step.id
if output_step_id not in self.outputs:
message = f"No outputs found for step id {output_step_id}, outputs are {self.outputs}"
raise Exception(message)
step_outputs = self.outputs[output_step_id]
if step_outputs is STEP_OUTPUT_DELAYED:
delayed_why = f"dependent step [{output_step_id}] delayed, so this step must be delayed"
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
output_name = connection.output_name
try:
replacement = step_outputs[output_name]
except KeyError:
# Must resolve.
template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s"
message = template % (output_name, step_outputs)
raise Exception(message)
if isinstance(replacement, model.HistoryDatasetCollectionAssociation):
if not replacement.collection.populated:
if not replacement.waiting_for_elements:
# If we are not waiting for elements, there was some
# problem creating the collection. Collection will never
# be populated.
# TODO: consider distinguish between cancelled and failed?
raise modules.CancelWorkflowEvaluation()
delayed_why = f"dependent collection [{replacement.id}] not yet populated with datasets"
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
if isinstance(replacement, model.DatasetCollection):
raise NotImplementedError
if not is_data and isinstance(
replacement, (model.HistoryDatasetAssociation,
model.HistoryDatasetCollectionAssociation)):
if isinstance(replacement, model.HistoryDatasetAssociation):
if replacement.is_pending:
raise modules.DelayedWorkflowEvaluation()
if not replacement.is_ok:
raise modules.CancelWorkflowEvaluation()
else:
if not replacement.collection.populated:
raise modules.DelayedWorkflowEvaluation()
pending = False
for dataset_instance in replacement.dataset_instances:
if dataset_instance.is_pending:
pending = True
elif not dataset_instance.is_ok:
raise modules.CancelWorkflowEvaluation()
if pending:
raise modules.DelayedWorkflowEvaluation()
return replacement
def replacement_for_connection(self, connection):
step_outputs = self.outputs[connection.output_step.id]
if step_outputs is STEP_OUTPUT_DELAYED:
raise modules.DelayedWorkflowEvaluation()
replacement = step_outputs[connection.output_name]
if isinstance(replacement, model.HistoryDatasetCollectionAssociation):
if not replacement.collection.populated:
if not replacement.collection.waiting_for_elements:
# If we are not waiting for elements, there was some
# problem creating the collection. Collection will never
# be populated.
# TODO: consider distinguish between cancelled and failed?
raise modules.CancelWorkflowEvaluation()
raise modules.DelayedWorkflowEvaluation()
return replacement
def get_replacement_workflow_output(self, workflow_output):
step = workflow_output.workflow_step
output_name = workflow_output.output_name
step_outputs = self.outputs[step.id]
if step_outputs is STEP_OUTPUT_DELAYED:
delayed_why = "depends on workflow output [%s] but that output has not been created yet" % output_name
raise modules.DelayedWorkflowEvaluation(why=delayed_why)
else:
return step_outputs[output_name]
def replacement_for_connection(self, connection, is_data=True):
output_step_id = connection.output_step.id
if output_step_id not in self.outputs:
template = "No outputs found for step id %s, outputs are %s"
message = template % (output_step_id, self.outputs)
raise Exception(message)
step_outputs = self.outputs[output_step_id]
if step_outputs is STEP_OUTPUT_DELAYED:
raise modules.DelayedWorkflowEvaluation()
output_name = connection.output_name
for key in step_outputs:
print "Output name: " + str(output_name)
#print "Output name, output1, result of object at ouput name: " + str(step_outputs["output1"])
#print "Result of object at output name: " + str(step_outputs[output_name])
#print "Object stuff: " + str(dir(step_outputs[key]))
print "Type of object: " + str(type(step_outputs))
try:
replacement = step_outputs[output_name]
except KeyError:
if is_data:
# Must resolve.
print "From run.py: " + str(step_outputs.viewvalues)
for key in step_outputs:
print "Key: " + key + " Value: " + str(step_outputs[key])
print "Object stuff: " + str(dir(step_outputs[key]))
template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s"
message = template % (output_name, step_outputs)
raise Exception(message)
else:
replacement = modules.NO_REPLACEMENT
if isinstance(replacement, model.HistoryDatasetCollectionAssociation):
if not replacement.collection.populated:
if not replacement.collection.waiting_for_elements:
# If we are not waiting for elements, there was some
# problem creating the collection. Collection will never
# be populated.
# TODO: consider distinguish between cancelled and failed?
raise modules.CancelWorkflowEvaluation()
raise modules.DelayedWorkflowEvaluation()
return replacement
def __check_implicitly_dependent_step( self, output_id ):
step_invocations = self.workflow_invocation.step_invocations_for_step_id( output_id )
# No steps created yet - have to delay evaluation.
if not step_invocations:
raise modules.DelayedWorkflowEvaluation()
for step_invocation in step_invocations:
job = step_invocation.job
if job:
# At least one job in incomplete.
if not job.finished:
raise modules.DelayedWorkflowEvaluation()
if job.state != job.states.OK:
raise modules.CancelWorkflowEvaluation()
else:
# TODO: Handle implicit dependency on stuff like
# pause steps.
pass
def replacement_for_connection( self, connection ):
step_outputs = self.outputs[ connection.output_step.id ]
if step_outputs is STEP_OUTPUT_DELAYED:
raise modules.DelayedWorkflowEvaluation()
output_name = connection.output_name
try:
replacement = step_outputs[ output_name ]
except KeyError:
template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s"
message = template % ( output_name, step_outputs )
raise Exception( message )
if isinstance( replacement, model.HistoryDatasetCollectionAssociation ):
if not replacement.collection.populated:
if not replacement.collection.waiting_for_elements:
# If we are not waiting for elements, there was some
# problem creating the collection. Collection will never
# be populated.
# TODO: consider distinguish between cancelled and failed?
raise modules.CancelWorkflowEvaluation()
raise modules.DelayedWorkflowEvaluation()
return replacement
def replacement_for_connection(self, connection):
step_outputs = self.outputs[connection.output_step.id]
if step_outputs is STEP_OUTPUT_DELAYED:
raise modules.DelayedWorkflowEvaluation()
return step_outputs[connection.output_name]