class ModuleInstance( AttributeCollection ): def __init__( self, name, obj = None, parent = None ): AttributeCollection.__init__( self ) self.instance_obj = None # used for the interpretation only self.parent = parent if obj == None: self.parameters = ParameterCollection() elif isinstance( obj, ModuleInstance ) or isinstance( obj, ModuleDefinition ): if name == None: self.setName( obj.getName() ) else: self.setName( name ) self.setType( obj.getType() ) self.setDescrShort( obj.getDescrShort() ) self.parameters = ParameterCollection( obj.parameters ) elif isinstance( obj, ParameterCollection ): # set attributes self.setName( name ) self.setType( "" ) self.setDescrShort( "" ) self.parameters = ParameterCollection( obj ) elif obj != None: raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) ) def createCode( self, ind = 2 ): str_ = indent( ind ) + self.getName() + ' = ' + self.getType() + '()\n' str_ = str_ + self.parameters.createParametersCode( ind, self.getName() ) str_ = str_ + indent( ind ) + self.getName() + '.execute()\n\n' return str_ def __str__( self ): return str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__() def toXML( self ): ret = '<ModuleInstance>\n' ret = ret + AttributeCollection.toXML( self ) ret = ret + self.parameters.toXML() ret = ret + '</ModuleInstance>\n' return ret def execute( self, step_parameters, definitions ): # print 'Executing ModuleInstance ',self.getName(),'of type',self.getType() self.instance_obj = definitions[self.getType()].main_class_obj() # creating instance self.parameters.execute( self.getName() ) self.instance_obj.execute2()
class StepInstance( AttributeCollection ): def __init__( self, name, obj = None, parent = None ): AttributeCollection.__init__( self ) self.parent = None if obj == None: self.parameters = ParameterCollection() elif isinstance( obj, StepInstance ) or isinstance( obj, StepDefinition ): if name == None: self.setName( obj.getName() ) else: self.setName( name ) self.setType( obj.getType() ) self.setDescrShort( obj.getDescrShort() ) self.parameters = ParameterCollection( obj.parameters ) elif ( obj == None ) or isinstance( obj, ParameterCollection ): # set attributes self.setName( name ) self.setType( "" ) self.setDescrShort( "" ) self.parameters = ParameterCollection( obj ) elif obj != None: raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) ) self.step_commons = {} self.stepStatus = S_OK() def resolveGlobalVars( self, step_definitions, wf_parameters ): ''' Resolve parameter values defined in the @{<variable>} form ''' self.parameters.resolveGlobalVars( wf_parameters ) module_instance_number = 0 for inst in step_definitions[self.getType()].module_instances: module_instance_number = module_instance_number + 1 if not inst.parameters.find( "MODULE_NUMBER" ): inst.parameters.append( Parameter( "MODULE_NUMBER", "%s" % module_instance_number, "string", "", "", True, False, "ModuleInstance number within the Step" ) ) if not inst.parameters.find( "MODULE_INSTANCE_NAME" ): inst.parameters.append( Parameter( "MODULE_INSTANCE_NAME", inst.getName(), "string", "", "", True, False, "Name of the ModuleInstance within the Step" ) ) if not inst.parameters.find( "MODULE_DEFINITION_NAME" ): inst.parameters.append( Parameter( "MODULE_DEFINITION_NAME", inst.getType(), "string", "", "", True, False, "Type of the ModuleInstance within the Step" ) ) if not inst.parameters.find( "JOB_ID" ): inst.parameters.append( Parameter( "JOB_ID", "", "string", "self", "JOB_ID", True, False, "Job ID within a Production as a string" ) ) if not inst.parameters.find( "PRODUCTION_ID" ): inst.parameters.append( Parameter( "PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID", True, False, "Production ID as a string" ) ) if not inst.parameters.find( "STEP_NUMBER" ): inst.parameters.append( Parameter( "STEP_NUMBER", "", "string", "self", "STEP_NUMBER", True, False, "Step instance number within the Workflow" ) ) if not inst.parameters.find( "STEP_ID" ): inst.parameters.append( Parameter( "STEP_ID", "", "string", "self", "STEP_NUMBER", True, False, "Step ID within the Workflow" ) ) inst.resolveGlobalVars( wf_parameters, self.parameters ) def createCode( self, ind = 2 ): ''' Create the Step code ''' str_ = indent( ind ) + self.getName() + ' = ' + self.getType() + '()\n' str_ = str_ + self.parameters.createParametersCode( ind, self.getName() ) str_ = str_ + indent( ind ) + self.getName() + '.execute()\n\n' return str_ def __str__( self ): ''' Step string representation ''' return str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__() def toXML( self ): ''' Generate the Step XML representation ''' ret = '<StepInstance>\n' ret = ret + AttributeCollection.toXML( self ) ret = ret + self.parameters.toXML() ret = ret + '</StepInstance>\n' return ret def setWorkflowCommons( self, wf ): ''' Add reference to the collection of the common tools ''' self.workflow_commons = wf def execute( self, step_exec_attr, definitions ): ''' Step execution method. step_exec_attr is array to hold parameters belong to this Step, filled above in the workflow ''' print 'Executing StepInstance', self.getName(), 'of type', self.getType(), definitions.keys() # Report the Application state if the coresponding tool is supplied if self.workflow_commons.has_key( 'JobReport' ): if self.parent.workflowStatus['OK']: result = self.workflow_commons['JobReport'].setApplicationStatus( 'Executing ' + self.getName() ) # Prepare Step statistics evaluation self.step_commons['StartTime'] = time.time() self.step_commons['StartStats'] = os.times() step_def = definitions[self.getType()] step_exec_modules = {} error_message = '' for mod_inst in step_def.module_instances: mod_inst_name = mod_inst.getName() mod_inst_type = mod_inst.getType() # print "StepInstance creating module instance ", mod_inst_name, " of type", mod_inst.getType() step_exec_modules[mod_inst_name] = \ step_def.parent.module_definitions[mod_inst_type].main_class_obj() # creating instance # Resolve all the linked parameter values for parameter in mod_inst.parameters: if parameter.preExecute(): # print '>>>> Input', parameter if parameter.isLinked(): # print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter() if parameter.getLinkedModule() == 'self': # tale value form the step_dict setattr( step_exec_modules[mod_inst_name], parameter.getName(), step_exec_attr[parameter.getLinkedParameter()] ) else: setattr( step_exec_modules[mod_inst_name], parameter.getName(), getattr( step_exec_modules[parameter.getLinkedModule()], parameter.getLinkedParameter() ) ) else: # print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getValue() setattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() ) # print 'Step Input Parameter:', parameter.getName(), getattr( step_exec_modules[mod_inst_name], parameter.getName() ) # Set reference to the workflow and step common tools setattr( step_exec_modules[mod_inst_name], 'workflow_commons', self.parent.workflow_commons ) setattr( step_exec_modules[mod_inst_name], 'step_commons', self.step_commons ) setattr( step_exec_modules[mod_inst_name], 'stepStatus', self.stepStatus ) setattr( step_exec_modules[mod_inst_name], 'workflowStatus', self.parent.workflowStatus ) try: result = step_exec_modules[mod_inst_name].execute() if not result['OK']: if self.stepStatus['OK']: error_message = result['Message'] if self.workflow_commons.has_key( 'JobReport' ): if self.parent.workflowStatus['OK']: resultStatus = self.workflow_commons['JobReport'].setApplicationStatus( error_message ) self.stepStatus = S_ERROR( result['Message'] ) else: for parameter in mod_inst.parameters: if parameter.isOutput(): # print '<<<< Output', parameter if parameter.isLinked(): # print "ModuleInstance self ." + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter() if parameter.getLinkedModule() == 'self': # this is not supposed to happen print "Warning! Module OUTPUT attribute", parameter.getName(), print "refer to the attribute of the same module", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName() ) step_exec_attr[parameter.getName()] = getattr( step_exec_modules[mod_inst_name], parameter.getLinkedParameter(), parameter.getValue() ) # print " OUT", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() ) else: # print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], parameter.getLinkedParameter() step_exec_attr[parameter.getName()] = \ getattr( step_exec_modules[parameter.getLinkedModule()], parameter.getLinkedParameter() ) else: # This also does not make sense - we can give a warning print "Warning! Module OUTPUT attribute ", parameter.getName(), print "assigned constant", parameter.getValue() # print "StepInstance self." + parameter.getName(), '=', parameter.getValue() step_exec_attr[parameter.getName()] = parameter.getValue() # print 'Module Output Parameter:', parameter.getName(), step_exec_attr[parameter.getName()] # Get output values to the step_commons dictionary for key in result.keys(): if key != "OK": if key != "Value": self.step_commons[key] = result[key] elif type( result['Value'] ) == types.DictType: for vkey in result['Value'].keys(): self.step_commons[vkey] = result['Value'][vkey] except Exception, x: print "Exception while module execution" print "Module", mod_inst_name, mod_inst.getType() print str( x ) exc = sys.exc_info() exc_type = exc[0] value = exc[1] print "== EXCEPTION ==\n%s: %s\n\n%s===============" % ( exc_type, value, "\n".join( traceback.format_tb( exc[2] ) ) ) print "Step status: ", self.stepStatus print "Workflow status: ", self.parent.workflowStatus if self.stepStatus['OK']: # This is the error that caused the workflow disruption # report it to the WMS error_message = 'Exception while %s module execution: %s' % ( mod_inst_name, str( x ) ) if self.workflow_commons.has_key( 'JobReport' ): if self.parent.workflowStatus['OK']: result = self.workflow_commons['JobReport'].setApplicationStatus( 'Exception in %s module' % mod_inst_name ) self.stepStatus = S_ERROR( error_message ) # now we need to copy output values to the STEP!!! parameters for st_parameter in self.parameters: if st_parameter.isOutput(): # print '<< Output', st_parameter if st_parameter.isLinked(): # print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getLinkedModule() + '.' + st_parameter.getLinkedParameter() if st_parameter.getLinkedModule() == 'self': # this is not supposed to happen print "Warning! Step OUTPUT attribute", st_parameter.getName(), print "refer to the attribute of the same step", st_parameter.getLinkedParameter(), step_exec_attr[st_parameter.getLinkedParameter()] step_exec_attr[st_parameter.getName()] = step_exec_attr[st_parameter.getLinkedParameter()] else: # print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter() step_exec_attr[st_parameter.getName()] = \ getattr( step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter() ) setattr( self, st_parameter.getName(), step_exec_attr[st_parameter.getName()] ) else: # This also does not make sense - we can give a warning print "Warning! Step OUTPUT attribute ", st_parameter.getName(), print "assigned constant", st_parameter.getValue() # print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getValue() step_exec_attr[st_parameter.getName()] = st_parameter.getValue() print 'Step Output', st_parameter.getName(), '=', step_exec_attr[st_parameter.getName()] # Return the result of the first failed module or S_OK if not self.stepStatus['OK']: return S_ERROR( error_message ) else: return S_OK( result['Value'] )
class Workflow(AttributeCollection): def __init__(self, obj=None, name=None): ''' Be aware that 1-st param is an obj not a name!!!! obj can me a string with XML representation or with filename also obj can be a Workflow or ParameterCollections ''' AttributeCollection.__init__(self) if (obj == None) or isinstance(obj, ParameterCollection): self.setName('notgiven') self.setType('') self.setDescrShort('') self.setDescription('') self.setOrigin('') self.setVersion(0.0) self.parameters = ParameterCollection(obj) self.step_instances = InstancesPool(self) self.step_definitions = DefinitionsPool(self) self.module_definitions = DefinitionsPool(self) elif isinstance(obj, Workflow): self.fromWorkflow(obj) elif isinstance(obj, str): self.parameters = ParameterCollection(None) self.step_instances = InstancesPool(self) self.step_definitions = DefinitionsPool(self) self.module_definitions = DefinitionsPool(self) # if obj is an XML string if obj.startswith('<'): fromXMLString(obj, self) else: fromXMLFile(obj, self) elif obj != None: raise TypeError('Can not create object type ' + str(type(self)) + ' from the ' + str(type(obj))) if name: self.setName(name) self.workflow_commons = {} self.workflowStatus = S_OK() def fromWorkflow(self, obj): self.setName(obj.getName()) self.setType(obj.getType()) self.setDescrShort(obj.getDescrShort()) self.setDescription(obj.getDescription()) self.setOrigin(obj.getOrigin()) self.setVersion(obj.getVersion()) # copy instances and definitions self.parameters = ParameterCollection(obj.parameters) self.module_definitions = DefinitionsPool(self, obj.module_definitions) self.step_instances = InstancesPool(self, obj.step_instances) self.step_definitions = DefinitionsPool(self, obj.step_definitions) def __str__(self): '''Creates a string representation of itself ''' ret = str(self.getName()) + ':\n' + AttributeCollection.__str__( self) + self.parameters.__str__() ret = ret + str(self.step_definitions) ret = ret + str(self.step_instances) ret = ret + str(self.module_definitions) return ret def toXML(self): '''Creates an XML representation of itself ''' # THIS is very important that Definitions should be written before instances ret = '<Workflow>\n' ret = ret + AttributeCollection.toXML(self) ret = ret + self.parameters.toXML() ret = ret + self.module_definitions.toXML() ret = ret + self.step_definitions.toXML() ret = ret + self.step_instances.toXML() ret = ret + '</Workflow>\n' return ret def toXMLFile(self, outFile): if os.path.exists(outFile): os.remove(outFile) xmlfile = open(outFile, 'w') xmlfile.write(self.toXML()) xmlfile.close() def addTool(self, name, tool): ''' Add an object that will be available in all the modules to perform some operations. For example, a state reporting facility. ''' self.workflow_commons[name] = tool def addStep(self, step): # this is a VERY important piece of code # we have to join all Modules definition from all added steps in the single dictionary # and we have to share this dictionary between all included steps # we also have to check versions of the modules and instances for type_ in step.module_definitions.keys(): #if self.module_definitions.has_key(type): # we have the same ModuleDefinition in 2 places # we need to find way to synchronise it #print "Workflow:addStep - we need to write ModuleDefinitions synchronisation code" #else: # new module - just append it if not self.module_definitions.has_key(type_): self.module_definitions.append(step.module_definitions[type_]) self.step_definitions.append(step) del step.module_definitions # we need to clean up all unwanted definitions step.module_definitions = None return step def addModule(self, module): # KGG We need to add code to update existing modules self.module_definitions.append(module) return module def createStepInstance(self, type_, name): ''' Creates step instance of type 'type' with the name 'name' ''' if self.step_definitions.has_key(type_): stepi = StepInstance(name, self.step_definitions[type_]) self.step_instances.append(stepi) return stepi else: raise KeyError('Can not find StepDefinition ' + type_ + ' to create StepInstrance ' + name) def removeStepInstance(self, name): self.instances[name].setParents(None) self.instances.delete(name) def updateParents(self): self.module_definitions.updateParents(self) self.step_instances.updateParents(self) self.step_definitions.updateParents(self) def resolveGlobalVars(self): ''' This method will create global parameter list and then will resolve all instances of @{VARNAME} Be aware that parameters of that type are GLOBAL!!! are string and can not be dynamically change The scope: the resolution of that parameters apply from lower to upper object, for example if parameter use in module, then it checks module, then step, then workflow Comment: If varible linked it should not be used in a global list''' # reenforced global parameters on the level of Workflow if not self.parameters.find("PRODUCTION_ID"): self.parameters.append( Parameter( "PRODUCTION_ID", "00000000", "string", "", "", True, False, "Transformation ID taken from the ProductionManager")) if not self.parameters.find("JOB_ID"): self.parameters.append( Parameter( "JOB_ID", "00000000", "string", "", "", True, False, "Job ID within Tranformationtaken from the ProductionManager" )) self.parameters.resolveGlobalVars() step_instance_number = 0 for inst in self.step_instances: # for each step instance we can define STEP_NUMBER step_instance_number = step_instance_number + 1 if not inst.parameters.find("STEP_NUMBER"): inst.parameters.append( Parameter( "STEP_NUMBER", "%s" % step_instance_number, "string", "", "", True, False, "Number of the StepInstance within the Workflow")) if not inst.parameters.find("STEP_ID"): prod_ID = self.parameters.find("PRODUCTION_ID").getValue() job_ID = self.parameters.find("JOB_ID").getValue() inst.parameters.append( Parameter( "STEP_ID", "%s_%s_%d" % (prod_ID, job_ID, step_instance_number), "string", "", "", True, False, "Step instance ID")) if not inst.parameters.find("STEP_INSTANCE_NAME"): inst.parameters.append( Parameter("STEP_INSTANCE_NAME", inst.getName(), "string", "", "", True, False, "Name of the StepInstance within the Workflow")) if not inst.parameters.find("STEP_DEFINITION_NAME"): inst.parameters.append( Parameter("STEP_DEFINITION_NAME", inst.getType(), "string", "", "", True, False, "Type of the StepInstance within the Workflow")) if not inst.parameters.find("JOB_ID"): inst.parameters.append( Parameter("JOB_ID", "", "string", "self", "JOB_ID", True, False, "Type of the StepInstance within the Workflow")) if not inst.parameters.find("PRODUCTION_ID"): inst.parameters.append( Parameter("PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID", True, False, "Type of the StepInstance within the Workflow")) inst.resolveGlobalVars(self.step_definitions, self.parameters) def createCode(self, combine_steps=False): self.resolveGlobalVars() str_ = '' str_ = str_ + self.module_definitions.createCode() str_ = str_ + self.step_definitions.createCode() str_ = str_ + "\nclass job:\n" str_ = str_ + indent(1) + 'def execute(self):\n' #str_=str_+indent(2)+'# flush self.step_instances\n' str_ = str_ + self.step_instances.createCode() # it seems we do not need it on this level str_ = str_ + indent(2) + '# output assignment\n' for v in self.parameters: if v.isOutput(): str_ = str_ + v.createParameterCode(2, 'self') str_ = str_ + '\nj=job()\n' str_ = str_ + self.parameters.createParametersCode(0, 'j') str_ = str_ + 'j.execute()' return str_ def showCode(self, combine_steps=False): str_ = '' str_ = str_ + self.module_definitions.createCode() str_ = str_ + self.step_definitions.createCode() str_ = str_ + "\nclass job:\n" str_ = str_ + indent(1) + 'def execute(self):\n' #str_=str_+indent(2)+'# flush self.step_instances\n' str_ = str_ + self.step_instances.createCode() # it seems we do not need it on this level str_ = str_ + indent(2) + '# output assignment\n' for v in self.parameters: if v.isOutput(): str_ = str_ + v.createParameterCode(2, 'self') str_ = str_ + '\nj=job()\n' str_ = str_ + self.parameters.createParametersCode(0, 'j') str_ = str_ + 'j.execute()' return str_ def execute(self): self.resolveGlobalVars() # define workflow attributes wf_exec_attr = { } # dictianary with the WF attributes, used to resolve links to self.attrname for wf_parameter in self.parameters: # parameters shall see objects in the current scope order to resolve links if wf_parameter.preExecute(): # for parm which not just outputs # print 'Input', wf_parameter if wf_parameter.isLinked(): # print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter() if wf_parameter.getLinkedModule() == 'self': # this is not suppose to happen print "Warning! Job attribute ", wf_parameter.getName( ), "refers to the attribute of the same workflow", wf_parameter.getLinkedParameter( ) wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[ wf_parameter.getLinkedParameter()] else: wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[ wf_parameter.getLinkedModule()][ wf_parameter.getLinkedParameter()] else: # print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getValue() wf_exec_attr[ wf_parameter.getName()] = wf_parameter.getValue() # Put all the workflow parameters into the workflow_commons dictionary self.workflow_commons[ wf_parameter.getName()] = wf_parameter.getValue() self.module_definitions.loadCode( ) # loading Module classes into current python scope #wf_exec_steps will be dictionary of dictionaries [step instance name][parameter name] # used as dictionary of step instances to carry parameters wf_exec_steps = {} # print 'Executing Workflow',self.getType() error_message = '' step_result = '' for step_inst in self.step_instances: step_inst_name = step_inst.getName() wf_exec_steps[step_inst_name] = {} # step_inst_type = step_inst.getType() # print "WorkflowInstance creating Step instance ",step_inst_name," of type", step_inst_type for parameter in step_inst.parameters: if parameter.preExecute(): # print '>> Input', parameter if parameter.isLinked(): # print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter() if parameter.getLinkedModule() == 'self': # tale value form the step_dict wf_exec_steps[step_inst_name][parameter.getName( )] = wf_exec_attr[parameter.getLinkedParameter()] else: # print wf_exec_steps[parameter.getLinkedModule()].keys() wf_exec_steps[step_inst_name][parameter.getName( )] = wf_exec_steps[parameter.getLinkedModule()][ parameter.getLinkedParameter()] else: # print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getValue() wf_exec_steps[step_inst_name][ parameter.getName()] = parameter.getValue() # In the step_commons all parameters are added, both Input and Output ones. step_inst.step_commons[ parameter.getName()] = parameter.getValue() resolveVariables(wf_exec_steps[step_inst_name]) # Set proper values for all Input Parameters for key, value in wf_exec_steps[step_inst_name].items(): step_inst.step_commons[key] = value step_inst.setParent(self) step_inst.setWorkflowCommons(self.workflow_commons) result = step_inst.execute(wf_exec_steps[step_inst_name], self.step_definitions) if not result['OK']: if self.workflowStatus['OK']: error_message = result['Message'] self.workflowStatus = S_ERROR(result['Message']) if result.has_key('Value'): step_result = result['Value'] # now we need to copy output values to the STEP!!! parameters #print "WorkflowInstance output assignment" for wf_parameter in self.parameters: if wf_parameter.isOutput(): if wf_parameter.isLinked(): # print "WorkflowInstance self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter() if wf_parameter.getLinkedModule() == 'self': # this is not suppose to happen print "Warning! Workflow OUTPUT attribute ", wf_parameter.getName( ), "refer on the attribute of the same workflow", wf_parameter.getLinkedParameter( ) wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[ wf_parameter.getLinkedParameter()] else: wf_exec_attr[wf_parameter.getName()] = wf_exec_steps[ wf_parameter.getLinkedModule()][ wf_parameter.getLinkedParameter()] else: # it is also does not make sense - we can produce warning print "Warning! Workflow OUTPUT attribute", wf_parameter.getName( ), "assigned constant", wf_parameter.getValue() # print "WorkflowInstance self."+ wf_parameter.getName(),'=',wf_parameter.getValue() wf_exec_attr[ wf_parameter.getName()] = wf_parameter.getValue() setattr(self, wf_parameter.getName(), wf_exec_attr[wf_parameter.getName()]) # Return the result of the first failed step or S_OK if not self.workflowStatus['OK']: return S_ERROR(error_message) else: return S_OK(step_result)
class Workflow( AttributeCollection ): def __init__( self, obj = None, name = None ): ''' Be aware that 1-st param is an obj not a name!!!! obj can me a string with XML representation or with filename also obj can be a Workflow or ParameterCollections ''' AttributeCollection.__init__( self ) if ( obj == None ) or isinstance( obj, ParameterCollection ): self.setName( 'notgiven' ) self.setType( '' ) self.setDescrShort( '' ) self.setDescription( '' ) self.setOrigin( '' ) self.setVersion( 0.0 ) self.parameters = ParameterCollection( obj ) self.step_instances = InstancesPool( self ) self.step_definitions = DefinitionsPool( self ) self.module_definitions = DefinitionsPool( self ) elif isinstance( obj, Workflow ): self.fromWorkflow( obj ) elif isinstance( obj, str ): self.parameters = ParameterCollection( None ) self.step_instances = InstancesPool( self ) self.step_definitions = DefinitionsPool( self ) self.module_definitions = DefinitionsPool( self ) # if obj is an XML string if obj.startswith( '<' ): fromXMLString( obj, self ) else: fromXMLFile( obj, self ) elif obj != None: raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) ) if name : self.setName( name ) self.workflow_commons = {} self.workflowStatus = S_OK() def fromWorkflow( self, obj ): self.setName( obj.getName() ) self.setType( obj.getType() ) self.setDescrShort( obj.getDescrShort() ) self.setDescription( obj.getDescription() ) self.setOrigin( obj.getOrigin() ) self.setVersion( obj.getVersion() ) # copy instances and definitions self.parameters = ParameterCollection( obj.parameters ) self.module_definitions = DefinitionsPool( self, obj.module_definitions ) self.step_instances = InstancesPool( self, obj.step_instances ) self.step_definitions = DefinitionsPool( self, obj.step_definitions ) def __str__( self ): '''Creates a string representation of itself ''' ret = str( self.getName() ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__() ret = ret + str( self.step_definitions ) ret = ret + str( self.step_instances ) ret = ret + str( self.module_definitions ) return ret def toXML( self ): '''Creates an XML representation of itself ''' # THIS is very important that Definitions should be written before instances ret = '<Workflow>\n' ret = ret + AttributeCollection.toXML( self ) ret = ret + self.parameters.toXML() ret = ret + self.module_definitions.toXML() ret = ret + self.step_definitions.toXML() ret = ret + self.step_instances.toXML() ret = ret + '</Workflow>\n' return ret def toXMLFile( self, outFile ): if os.path.exists( outFile ): os.remove( outFile ) xmlfile = open( outFile, 'w' ) xmlfile.write( self.toXML() ) xmlfile.close() def addTool( self, name, tool ): ''' Add an object that will be available in all the modules to perform some operations. For example, a state reporting facility. ''' self.workflow_commons[name] = tool def addStep( self, step ): # this is a VERY important piece of code # we have to join all Modules definition from all added steps in the single dictionary # and we have to share this dictionary between all included steps # we also have to check versions of the modules and instances for type_ in step.module_definitions.keys(): #if self.module_definitions.has_key(type): # we have the same ModuleDefinition in 2 places # we need to find way to synchronise it #print "Workflow:addStep - we need to write ModuleDefinitions synchronisation code" #else: # new module - just append it if not self.module_definitions.has_key( type_ ): self.module_definitions.append( step.module_definitions[type_] ) self.step_definitions.append( step ) del step.module_definitions # we need to clean up all unwanted definitions step.module_definitions = None return step def addModule( self, module ): # KGG We need to add code to update existing modules self.module_definitions.append( module ) return module def createStepInstance( self, type_, name ): ''' Creates step instance of type 'type' with the name 'name' ''' if self.step_definitions.has_key( type_ ): stepi = StepInstance( name, self.step_definitions[type_] ) self.step_instances.append( stepi ) return stepi else: raise KeyError( 'Can not find StepDefinition ' + type_ + ' to create StepInstrance ' + name ) def removeStepInstance( self, name ): self.instances[name].setParents( None ) self.instances.delete( name ) def updateParents( self ): self.module_definitions.updateParents( self ) self.step_instances.updateParents( self ) self.step_definitions.updateParents( self ) def resolveGlobalVars( self ): ''' This method will create global parameter list and then will resolve all instances of @{VARNAME} Be aware that parameters of that type are GLOBAL!!! are string and can not be dynamically change The scope: the resolution of that parameters apply from lower to upper object, for example if parameter use in module, then it checks module, then step, then workflow Comment: If varible linked it should not be used in a global list''' # reenforced global parameters on the level of Workflow if not self.parameters.find( "PRODUCTION_ID" ): self.parameters.append( Parameter( "PRODUCTION_ID", "00000000", "string", "", "", True, False, "Transformation ID taken from the ProductionManager" ) ) if not self.parameters.find( "JOB_ID" ): self.parameters.append( Parameter( "JOB_ID", "00000000", "string", "", "", True, False, "Job ID within Tranformationtaken from the ProductionManager" ) ) self.parameters.resolveGlobalVars() step_instance_number = 0 for inst in self.step_instances: # for each step instance we can define STEP_NUMBER step_instance_number = step_instance_number + 1 if not inst.parameters.find( "STEP_NUMBER" ): inst.parameters.append( Parameter( "STEP_NUMBER", "%s" % step_instance_number, "string", "", "", True, False, "Number of the StepInstance within the Workflow" ) ) if not inst.parameters.find( "STEP_ID" ): prod_ID = self.parameters.find( "PRODUCTION_ID" ).getValue() job_ID = self.parameters.find( "JOB_ID" ).getValue() inst.parameters.append( Parameter( "STEP_ID", "%s_%s_%d" % ( prod_ID, job_ID, step_instance_number ), "string", "", "", True, False, "Step instance ID" ) ) if not inst.parameters.find( "STEP_INSTANCE_NAME" ): inst.parameters.append( Parameter( "STEP_INSTANCE_NAME", inst.getName(), "string", "", "", True, False, "Name of the StepInstance within the Workflow" ) ) if not inst.parameters.find( "STEP_DEFINITION_NAME" ): inst.parameters.append( Parameter( "STEP_DEFINITION_NAME", inst.getType(), "string", "", "", True, False, "Type of the StepInstance within the Workflow" ) ) if not inst.parameters.find( "JOB_ID" ): inst.parameters.append( Parameter( "JOB_ID", "", "string", "self", "JOB_ID", True, False, "Type of the StepInstance within the Workflow" ) ) if not inst.parameters.find( "PRODUCTION_ID" ): inst.parameters.append( Parameter( "PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID", True, False, "Type of the StepInstance within the Workflow" ) ) inst.resolveGlobalVars( self.step_definitions, self.parameters ) def createCode( self, combine_steps = False ): self.resolveGlobalVars() str_ = '' str_ = str_ + self.module_definitions.createCode() str_ = str_ + self.step_definitions.createCode() str_ = str_ + "\nclass job:\n" str_ = str_ + indent( 1 ) + 'def execute(self):\n' #str_=str_+indent(2)+'# flush self.step_instances\n' str_ = str_ + self.step_instances.createCode() # it seems we do not need it on this level str_ = str_ + indent( 2 ) + '# output assignment\n' for v in self.parameters: if v.isOutput(): str_ = str_ + v.createParameterCode( 2, 'self' ) str_ = str_ + '\nj=job()\n' str_ = str_ + self.parameters.createParametersCode( 0, 'j' ) str_ = str_ + 'j.execute()' return str_ def showCode( self, combine_steps = False ): str_ = '' str_ = str_ + self.module_definitions.createCode() str_ = str_ + self.step_definitions.createCode() str_ = str_ + "\nclass job:\n" str_ = str_ + indent( 1 ) + 'def execute(self):\n' #str_=str_+indent(2)+'# flush self.step_instances\n' str_ = str_ + self.step_instances.createCode() # it seems we do not need it on this level str_ = str_ + indent( 2 ) + '# output assignment\n' for v in self.parameters: if v.isOutput(): str_ = str_ + v.createParameterCode( 2, 'self' ) str_ = str_ + '\nj=job()\n' str_ = str_ + self.parameters.createParametersCode( 0, 'j' ) str_ = str_ + 'j.execute()' return str_ def execute( self ): self.resolveGlobalVars() # define workflow attributes wf_exec_attr = {} # dictianary with the WF attributes, used to resolve links to self.attrname for wf_parameter in self.parameters: # parameters shall see objects in the current scope order to resolve links if wf_parameter.preExecute(): # for parm which not just outputs # print 'Input', wf_parameter if wf_parameter.isLinked(): # print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter() if wf_parameter.getLinkedModule() == 'self': # this is not suppose to happen print "Warning! Job attribute ", wf_parameter.getName(), "refers to the attribute of the same workflow", wf_parameter.getLinkedParameter() wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedParameter()] else: wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedModule()][wf_parameter.getLinkedParameter()] else: # print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getValue() wf_exec_attr[wf_parameter.getName()] = wf_parameter.getValue() # Put all the workflow parameters into the workflow_commons dictionary self.workflow_commons[wf_parameter.getName()] = wf_parameter.getValue() self.module_definitions.loadCode() # loading Module classes into current python scope #wf_exec_steps will be dictionary of dictionaries [step instance name][parameter name] # used as dictionary of step instances to carry parameters wf_exec_steps = {} # print 'Executing Workflow',self.getType() error_message = '' step_result = '' for step_inst in self.step_instances: step_inst_name = step_inst.getName() wf_exec_steps[step_inst_name] = {} # step_inst_type = step_inst.getType() # print "WorkflowInstance creating Step instance ",step_inst_name," of type", step_inst_type for parameter in step_inst.parameters: if parameter.preExecute(): # print '>> Input', parameter if parameter.isLinked(): # print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter() if parameter.getLinkedModule() == 'self': # tale value form the step_dict wf_exec_steps[step_inst_name][parameter.getName()] = wf_exec_attr[parameter.getLinkedParameter()] else: # print wf_exec_steps[parameter.getLinkedModule()].keys() wf_exec_steps[step_inst_name][parameter.getName()] = wf_exec_steps[parameter.getLinkedModule()][parameter.getLinkedParameter()] else: # print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getValue() wf_exec_steps[step_inst_name][parameter.getName()] = parameter.getValue() # In the step_commons all parameters are added, both Input and Output ones. step_inst.step_commons[parameter.getName()] = parameter.getValue() resolveVariables( wf_exec_steps[step_inst_name] ) # Set proper values for all Input Parameters for key, value in wf_exec_steps[step_inst_name].items(): step_inst.step_commons[key] = value step_inst.setParent( self ) step_inst.setWorkflowCommons( self.workflow_commons ) result = step_inst.execute( wf_exec_steps[step_inst_name], self.step_definitions ) if not result['OK']: if self.workflowStatus['OK']: error_message = result['Message'] self.workflowStatus = S_ERROR( result['Message'] ) if result.has_key( 'Value' ): step_result = result['Value'] # now we need to copy output values to the STEP!!! parameters #print "WorkflowInstance output assignment" for wf_parameter in self.parameters: if wf_parameter.isOutput(): if wf_parameter.isLinked(): # print "WorkflowInstance self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter() if wf_parameter.getLinkedModule() == 'self': # this is not suppose to happen print "Warning! Workflow OUTPUT attribute ", wf_parameter.getName(), "refer on the attribute of the same workflow", wf_parameter.getLinkedParameter() wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedParameter()] else: wf_exec_attr[wf_parameter.getName()] = wf_exec_steps[wf_parameter.getLinkedModule()][wf_parameter.getLinkedParameter()] else: # it is also does not make sense - we can produce warning print "Warning! Workflow OUTPUT attribute", wf_parameter.getName(), "assigned constant", wf_parameter.getValue() # print "WorkflowInstance self."+ wf_parameter.getName(),'=',wf_parameter.getValue() wf_exec_attr[wf_parameter.getName()] = wf_parameter.getValue() setattr( self, wf_parameter.getName(), wf_exec_attr[wf_parameter.getName()] ) # Return the result of the first failed step or S_OK if not self.workflowStatus['OK']: return S_ERROR( error_message ) else: return S_OK( step_result )
class StepInstance(AttributeCollection): def __init__(self, name, obj=None, parent=None): AttributeCollection.__init__(self) self.parent = None if obj == None: self.parameters = ParameterCollection() elif isinstance(obj, StepInstance) or isinstance(obj, StepDefinition): if name == None: self.setName(obj.getName()) else: self.setName(name) self.setType(obj.getType()) self.setDescrShort(obj.getDescrShort()) self.parameters = ParameterCollection(obj.parameters) elif (obj == None) or isinstance(obj, ParameterCollection): # set attributes self.setName(name) self.setType("") self.setDescrShort("") self.parameters = ParameterCollection(obj) elif obj != None: raise TypeError('Can not create object type ' + str(type(self)) + ' from the ' + str(type(obj))) self.step_commons = {} self.stepStatus = S_OK() def resolveGlobalVars(self, step_definitions, wf_parameters): ''' Resolve parameter values defined in the @{<variable>} form ''' self.parameters.resolveGlobalVars(wf_parameters) module_instance_number = 0 for inst in step_definitions[self.getType()].module_instances: module_instance_number = module_instance_number + 1 if not inst.parameters.find("MODULE_NUMBER"): inst.parameters.append( Parameter("MODULE_NUMBER", "%s" % module_instance_number, "string", "", "", True, False, "ModuleInstance number within the Step")) if not inst.parameters.find("MODULE_INSTANCE_NAME"): inst.parameters.append( Parameter("MODULE_INSTANCE_NAME", inst.getName(), "string", "", "", True, False, "Name of the ModuleInstance within the Step")) if not inst.parameters.find("MODULE_DEFINITION_NAME"): inst.parameters.append( Parameter("MODULE_DEFINITION_NAME", inst.getType(), "string", "", "", True, False, "Type of the ModuleInstance within the Step")) if not inst.parameters.find("JOB_ID"): inst.parameters.append( Parameter("JOB_ID", "", "string", "self", "JOB_ID", True, False, "Job ID within a Production as a string")) if not inst.parameters.find("PRODUCTION_ID"): inst.parameters.append( Parameter("PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID", True, False, "Production ID as a string")) if not inst.parameters.find("STEP_NUMBER"): inst.parameters.append( Parameter("STEP_NUMBER", "", "string", "self", "STEP_NUMBER", True, False, "Step instance number within the Workflow")) if not inst.parameters.find("STEP_ID"): inst.parameters.append( Parameter("STEP_ID", "", "string", "self", "STEP_NUMBER", True, False, "Step ID within the Workflow")) inst.resolveGlobalVars(wf_parameters, self.parameters) def createCode(self, ind=2): ''' Create the Step code ''' str_ = indent(ind) + self.getName() + ' = ' + self.getType() + '()\n' str_ = str_ + self.parameters.createParametersCode(ind, self.getName()) str_ = str_ + indent(ind) + self.getName() + '.execute()\n\n' return str_ def __str__(self): ''' Step string representation ''' return str(type(self)) + ':\n' + AttributeCollection.__str__( self) + self.parameters.__str__() def toXML(self): ''' Generate the Step XML representation ''' ret = '<StepInstance>\n' ret = ret + AttributeCollection.toXML(self) ret = ret + self.parameters.toXML() ret = ret + '</StepInstance>\n' return ret def setWorkflowCommons(self, wf): ''' Add reference to the collection of the common tools ''' self.workflow_commons = wf def execute(self, step_exec_attr, definitions): ''' Step execution method. step_exec_attr is array to hold parameters belong to this Step, filled above in the workflow ''' print 'Executing StepInstance', self.getName( ), 'of type', self.getType(), definitions.keys() # Report the Application state if the coresponding tool is supplied if self.workflow_commons.has_key('JobReport'): if self.parent.workflowStatus['OK']: result = self.workflow_commons[ 'JobReport'].setApplicationStatus('Executing ' + self.getName()) # Prepare Step statistics evaluation self.step_commons['StartTime'] = time.time() self.step_commons['StartStats'] = os.times() step_def = definitions[self.getType()] step_exec_modules = {} error_message = '' for mod_inst in step_def.module_instances: mod_inst_name = mod_inst.getName() mod_inst_type = mod_inst.getType() # print "StepInstance creating module instance ", mod_inst_name, " of type", mod_inst.getType() step_exec_modules[mod_inst_name] = \ step_def.parent.module_definitions[mod_inst_type].main_class_obj() # creating instance # Resolve all the linked parameter values for parameter in mod_inst.parameters: if parameter.preExecute(): # print '>>>> Input', parameter if parameter.isLinked(): # print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter() if parameter.getLinkedModule() == 'self': # tale value form the step_dict setattr( step_exec_modules[mod_inst_name], parameter.getName(), step_exec_attr[parameter.getLinkedParameter()]) else: setattr( step_exec_modules[mod_inst_name], parameter.getName(), getattr( step_exec_modules[ parameter.getLinkedModule()], parameter.getLinkedParameter())) else: # print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getValue() setattr(step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue()) # print 'Step Input Parameter:', parameter.getName(), getattr( step_exec_modules[mod_inst_name], parameter.getName() ) # Set reference to the workflow and step common tools setattr(step_exec_modules[mod_inst_name], 'workflow_commons', self.parent.workflow_commons) setattr(step_exec_modules[mod_inst_name], 'step_commons', self.step_commons) setattr(step_exec_modules[mod_inst_name], 'stepStatus', self.stepStatus) setattr(step_exec_modules[mod_inst_name], 'workflowStatus', self.parent.workflowStatus) try: result = step_exec_modules[mod_inst_name].execute() if not result['OK']: if self.stepStatus['OK']: error_message = result['Message'] if self.workflow_commons.has_key('JobReport'): if self.parent.workflowStatus['OK']: resultStatus = self.workflow_commons[ 'JobReport'].setApplicationStatus( error_message) self.stepStatus = S_ERROR(result['Message']) else: for parameter in mod_inst.parameters: if parameter.isOutput(): # print '<<<< Output', parameter if parameter.isLinked(): # print "ModuleInstance self ." + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter() if parameter.getLinkedModule() == 'self': # this is not supposed to happen print "Warning! Module OUTPUT attribute", parameter.getName( ), print "refer to the attribute of the same module", parameter.getLinkedParameter( ), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName()) step_exec_attr[ parameter.getName()] = getattr( step_exec_modules[mod_inst_name], parameter.getLinkedParameter(), parameter.getValue()) # print " OUT", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() ) else: # print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], parameter.getLinkedParameter() step_exec_attr[parameter.getName()] = \ getattr( step_exec_modules[parameter.getLinkedModule()], parameter.getLinkedParameter() ) else: # This also does not make sense - we can give a warning print "Warning! Module OUTPUT attribute ", parameter.getName( ), print "assigned constant", parameter.getValue() # print "StepInstance self." + parameter.getName(), '=', parameter.getValue() step_exec_attr[parameter.getName( )] = parameter.getValue() # print 'Module Output Parameter:', parameter.getName(), step_exec_attr[parameter.getName()] # Get output values to the step_commons dictionary for key in result.keys(): if key != "OK": if key != "Value": self.step_commons[key] = result[key] elif type(result['Value']) == types.DictType: for vkey in result['Value'].keys(): self.step_commons[vkey] = result['Value'][ vkey] except Exception, x: print "Exception while module execution" print "Module", mod_inst_name, mod_inst.getType() print str(x) exc = sys.exc_info() exc_type = exc[0] value = exc[1] print "== EXCEPTION ==\n%s: %s\n\n%s===============" % ( exc_type, value, "\n".join(traceback.format_tb(exc[2]))) print "Step status: ", self.stepStatus print "Workflow status: ", self.parent.workflowStatus if self.stepStatus['OK']: # This is the error that caused the workflow disruption # report it to the WMS error_message = 'Exception while %s module execution: %s' % ( mod_inst_name, str(x)) if self.workflow_commons.has_key('JobReport'): if self.parent.workflowStatus['OK']: result = self.workflow_commons[ 'JobReport'].setApplicationStatus( 'Exception in %s module' % mod_inst_name) self.stepStatus = S_ERROR(error_message) # now we need to copy output values to the STEP!!! parameters for st_parameter in self.parameters: if st_parameter.isOutput(): # print '<< Output', st_parameter if st_parameter.isLinked(): # print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getLinkedModule() + '.' + st_parameter.getLinkedParameter() if st_parameter.getLinkedModule() == 'self': # this is not supposed to happen print "Warning! Step OUTPUT attribute", st_parameter.getName( ), print "refer to the attribute of the same step", st_parameter.getLinkedParameter( ), step_exec_attr[st_parameter.getLinkedParameter()] step_exec_attr[st_parameter.getName( )] = step_exec_attr[st_parameter.getLinkedParameter()] else: # print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter() step_exec_attr[st_parameter.getName()] = \ getattr( step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter() ) setattr(self, st_parameter.getName(), step_exec_attr[st_parameter.getName()]) else: # This also does not make sense - we can give a warning print "Warning! Step OUTPUT attribute ", st_parameter.getName( ), print "assigned constant", st_parameter.getValue() # print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getValue() step_exec_attr[ st_parameter.getName()] = st_parameter.getValue() print 'Step Output', st_parameter.getName( ), '=', step_exec_attr[st_parameter.getName()] # Return the result of the first failed module or S_OK if not self.stepStatus['OK']: return S_ERROR(error_message) else: return S_OK(result['Value'])