def replacement_for_connection(self, connection, is_data=True): output_step_id = connection.output_step.id if output_step_id not in self.outputs: template = "No outputs found for step id %s, outputs are %s" message = template % (output_step_id, self.outputs) raise Exception(message) step_outputs = self.outputs[output_step_id] if step_outputs is STEP_OUTPUT_DELAYED: delayed_why = "dependent step [%s] delayed, so this step must be delayed" % output_step_id raise modules.DelayedWorkflowEvaluation(why=delayed_why) output_name = connection.output_name try: replacement = step_outputs[output_name] except KeyError: if is_data: # Must resolve. template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" message = template % (output_name, step_outputs) raise Exception(message) else: replacement = modules.NO_REPLACEMENT if isinstance(replacement, model.HistoryDatasetCollectionAssociation): if not replacement.collection.populated: if not replacement.collection.waiting_for_elements: # If we are not waiting for elements, there was some # problem creating the collection. Collection will never # be populated. # TODO: consider distinguish between cancelled and failed? raise modules.CancelWorkflowEvaluation() delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id raise modules.DelayedWorkflowEvaluation(why=delayed_why) return replacement
def __check_implicitly_dependent_step(self, output_id): step_invocation = self.workflow_invocation.step_invocation_for_step_id( output_id) # No steps created yet - have to delay evaluation. if not step_invocation: delayed_why = "depends on step [%s] but that step has not been invoked yet" % output_id raise modules.DelayedWorkflowEvaluation(why=delayed_why) if step_invocation.state != 'scheduled': delayed_why = "depends on step [%s] job has not finished scheduling yet" % output_id raise modules.DelayedWorkflowEvaluation(delayed_why) for job_assoc in step_invocation.jobs: job = job_assoc.job if job: # At least one job in incomplete. if not job.finished: delayed_why = "depends on step [%s] but one or more jobs created from that step have not finished yet" % output_id raise modules.DelayedWorkflowEvaluation(why=delayed_why) if job.state != job.states.OK: raise modules.CancelWorkflowEvaluation() else: # TODO: Handle implicit dependency on stuff like # pause steps. pass
def replacement_for_connection(self, connection, is_data=True): output_step_id = connection.output_step.id if output_step_id not in self.outputs: template = "No outputs found for step id %s, outputs are %s" message = template % (output_step_id, self.outputs) raise Exception(message) step_outputs = self.outputs[output_step_id] if step_outputs is STEP_OUTPUT_DELAYED: delayed_why = "dependent step [%s] delayed, so this step must be delayed" % output_step_id raise modules.DelayedWorkflowEvaluation(why=delayed_why) output_name = connection.output_name try: replacement = step_outputs[output_name] except KeyError: replacement = self.inputs_by_step_id.get(output_step_id) if connection.output_step.type == 'parameter_input' and output_step_id is not None: # FIXME: parameter_input step outputs should be properly recorded as step outputs, but for now we can # short-circuit and just pick the input value pass else: # Must resolve. template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" message = template % (output_name, step_outputs) raise Exception(message) if isinstance(replacement, model.HistoryDatasetCollectionAssociation): if not replacement.collection.populated: if not replacement.collection.waiting_for_elements: # If we are not waiting for elements, there was some # problem creating the collection. Collection will never # be populated. # TODO: consider distinguish between cancelled and failed? raise modules.CancelWorkflowEvaluation() delayed_why = "dependent collection [%s] not yet populated with datasets" % replacement.id raise modules.DelayedWorkflowEvaluation(why=delayed_why) data_inputs = (model.HistoryDatasetAssociation, model.HistoryDatasetCollectionAssociation, model.DatasetCollection) if not is_data and isinstance(replacement, data_inputs): if isinstance(replacement, model.HistoryDatasetAssociation): if replacement.is_pending: raise modules.DelayedWorkflowEvaluation() if not replacement.is_ok: raise modules.CancelWorkflowEvaluation() else: if not replacement.collection.populated: raise modules.DelayedWorkflowEvaluation() pending = False for dataset_instance in replacement.dataset_instances: if dataset_instance.is_pending: pending = True elif not dataset_instance.is_ok: raise modules.CancelWorkflowEvaluation() if pending: raise modules.DelayedWorkflowEvaluation() return replacement
def replacement_for_connection(self, connection, is_data=True): output_step_id = connection.output_step.id if output_step_id not in self.outputs: message = f"No outputs found for step id {output_step_id}, outputs are {self.outputs}" raise Exception(message) step_outputs = self.outputs[output_step_id] if step_outputs is STEP_OUTPUT_DELAYED: delayed_why = f"dependent step [{output_step_id}] delayed, so this step must be delayed" raise modules.DelayedWorkflowEvaluation(why=delayed_why) output_name = connection.output_name try: replacement = step_outputs[output_name] except KeyError: # Must resolve. template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" message = template % (output_name, step_outputs) raise Exception(message) if isinstance(replacement, model.HistoryDatasetCollectionAssociation): if not replacement.collection.populated: if not replacement.waiting_for_elements: # If we are not waiting for elements, there was some # problem creating the collection. Collection will never # be populated. # TODO: consider distinguish between cancelled and failed? raise modules.CancelWorkflowEvaluation() delayed_why = f"dependent collection [{replacement.id}] not yet populated with datasets" raise modules.DelayedWorkflowEvaluation(why=delayed_why) if isinstance(replacement, model.DatasetCollection): raise NotImplementedError if not is_data and isinstance( replacement, (model.HistoryDatasetAssociation, model.HistoryDatasetCollectionAssociation)): if isinstance(replacement, model.HistoryDatasetAssociation): if replacement.is_pending: raise modules.DelayedWorkflowEvaluation() if not replacement.is_ok: raise modules.CancelWorkflowEvaluation() else: if not replacement.collection.populated: raise modules.DelayedWorkflowEvaluation() pending = False for dataset_instance in replacement.dataset_instances: if dataset_instance.is_pending: pending = True elif not dataset_instance.is_ok: raise modules.CancelWorkflowEvaluation() if pending: raise modules.DelayedWorkflowEvaluation() return replacement
def replacement_for_connection(self, connection): step_outputs = self.outputs[connection.output_step.id] if step_outputs is STEP_OUTPUT_DELAYED: raise modules.DelayedWorkflowEvaluation() replacement = step_outputs[connection.output_name] if isinstance(replacement, model.HistoryDatasetCollectionAssociation): if not replacement.collection.populated: if not replacement.collection.waiting_for_elements: # If we are not waiting for elements, there was some # problem creating the collection. Collection will never # be populated. # TODO: consider distinguish between cancelled and failed? raise modules.CancelWorkflowEvaluation() raise modules.DelayedWorkflowEvaluation() return replacement
def get_replacement_workflow_output(self, workflow_output): step = workflow_output.workflow_step output_name = workflow_output.output_name step_outputs = self.outputs[step.id] if step_outputs is STEP_OUTPUT_DELAYED: delayed_why = "depends on workflow output [%s] but that output has not been created yet" % output_name raise modules.DelayedWorkflowEvaluation(why=delayed_why) else: return step_outputs[output_name]
def replacement_for_connection(self, connection, is_data=True): output_step_id = connection.output_step.id if output_step_id not in self.outputs: template = "No outputs found for step id %s, outputs are %s" message = template % (output_step_id, self.outputs) raise Exception(message) step_outputs = self.outputs[output_step_id] if step_outputs is STEP_OUTPUT_DELAYED: raise modules.DelayedWorkflowEvaluation() output_name = connection.output_name for key in step_outputs: print "Output name: " + str(output_name) #print "Output name, output1, result of object at ouput name: " + str(step_outputs["output1"]) #print "Result of object at output name: " + str(step_outputs[output_name]) #print "Object stuff: " + str(dir(step_outputs[key])) print "Type of object: " + str(type(step_outputs)) try: replacement = step_outputs[output_name] except KeyError: if is_data: # Must resolve. print "From run.py: " + str(step_outputs.viewvalues) for key in step_outputs: print "Key: " + key + " Value: " + str(step_outputs[key]) print "Object stuff: " + str(dir(step_outputs[key])) template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" message = template % (output_name, step_outputs) raise Exception(message) else: replacement = modules.NO_REPLACEMENT if isinstance(replacement, model.HistoryDatasetCollectionAssociation): if not replacement.collection.populated: if not replacement.collection.waiting_for_elements: # If we are not waiting for elements, there was some # problem creating the collection. Collection will never # be populated. # TODO: consider distinguish between cancelled and failed? raise modules.CancelWorkflowEvaluation() raise modules.DelayedWorkflowEvaluation() return replacement
def __check_implicitly_dependent_step( self, output_id ): step_invocations = self.workflow_invocation.step_invocations_for_step_id( output_id ) # No steps created yet - have to delay evaluation. if not step_invocations: raise modules.DelayedWorkflowEvaluation() for step_invocation in step_invocations: job = step_invocation.job if job: # At least one job in incomplete. if not job.finished: raise modules.DelayedWorkflowEvaluation() if job.state != job.states.OK: raise modules.CancelWorkflowEvaluation() else: # TODO: Handle implicit dependency on stuff like # pause steps. pass
def replacement_for_connection( self, connection ): step_outputs = self.outputs[ connection.output_step.id ] if step_outputs is STEP_OUTPUT_DELAYED: raise modules.DelayedWorkflowEvaluation() output_name = connection.output_name try: replacement = step_outputs[ output_name ] except KeyError: template = "Workflow evaluation problem - failed to find output_name %s in step_outputs %s" message = template % ( output_name, step_outputs ) raise Exception( message ) if isinstance( replacement, model.HistoryDatasetCollectionAssociation ): if not replacement.collection.populated: if not replacement.collection.waiting_for_elements: # If we are not waiting for elements, there was some # problem creating the collection. Collection will never # be populated. # TODO: consider distinguish between cancelled and failed? raise modules.CancelWorkflowEvaluation() raise modules.DelayedWorkflowEvaluation() return replacement
def replacement_for_connection(self, connection): step_outputs = self.outputs[connection.output_step.id] if step_outputs is STEP_OUTPUT_DELAYED: raise modules.DelayedWorkflowEvaluation() return step_outputs[connection.output_name]