def __init__( self, step_type = None, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setType( 'notgiven' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj ) # creating copy
self.module_instances = InstancesPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, StepDefinition ):
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
# copy instances and definitions
self.parameters = ParameterCollection( self, obj.parameters )
self.module_instances = InstancesPool( self, obj.module_instances )
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool( self. obj.module_definitions )
else:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if step_type :
self.setType( step_type )
def __init__( self, type_ = None, obj = None, parent = None ):
# we can create an object from another module
# or from the ParameterCollection
AttributeCollection.__init__( self )
self.main_class_obj = None # used for the interpretation only
# self.module_obj = None # used for the interpretation only
self.parent = parent
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setType( 'notgiven' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setRequired( '' )
self.setBody( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj ) # creating copy
elif isinstance( obj, ModuleDefinition ):
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setBody( obj.getBody() )
self.setRequired( obj.getRequired() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
self.parameters = ParameterCollection( obj.parameters )
else:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if type_ :
self.setType( type_ )
def __init__( self, name, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.parent = None
if obj == None:
self.parameters = ParameterCollection()
elif isinstance( obj, StepInstance ) or isinstance( obj, StepDefinition ):
if name == None:
self.setName( obj.getName() )
else:
self.setName( name )
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.parameters = ParameterCollection( obj.parameters )
elif ( obj == None ) or isinstance( obj, ParameterCollection ):
# set attributes
self.setName( name )
self.setType( "" )
self.setDescrShort( "" )
self.parameters = ParameterCollection( obj )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
self.step_commons = {}
self.stepStatus = S_OK()
def fromWorkflow(self, obj):
self.setName(obj.getName())
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(obj.parameters)
self.module_definitions = DefinitionsPool(self, obj.module_definitions)
self.step_instances = InstancesPool(self, obj.step_instances)
self.step_definitions = DefinitionsPool(self, obj.step_definitions)
class ModuleInstance( AttributeCollection ):
def __init__( self, name, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.instance_obj = None # used for the interpretation only
self.parent = parent
if obj == None:
self.parameters = ParameterCollection()
elif isinstance( obj, ModuleInstance ) or isinstance( obj, ModuleDefinition ):
if name == None:
self.setName( obj.getName() )
else:
self.setName( name )
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.parameters = ParameterCollection( obj.parameters )
elif isinstance( obj, ParameterCollection ):
# set attributes
self.setName( name )
self.setType( "" )
self.setDescrShort( "" )
self.parameters = ParameterCollection( obj )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
def createCode( self, ind = 2 ):
str_ = indent( ind ) + self.getName() + ' = ' + self.getType() + '()\n'
str_ = str_ + self.parameters.createParametersCode( ind, self.getName() )
str_ = str_ + indent( ind ) + self.getName() + '.execute()\n\n'
return str_
def __str__( self ):
return str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
def toXML( self ):
ret = '<ModuleInstance>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
ret = ret + '</ModuleInstance>\n'
return ret
def execute( self, step_parameters, definitions ):
# print 'Executing ModuleInstance ',self.getName(),'of type',self.getType()
self.instance_obj = definitions[self.getType()].main_class_obj() # creating instance
self.parameters.execute( self.getName() )
self.instance_obj.execute2()
def fromWorkflow( self, obj ): self.setName( obj.getName() ) self.setType( obj.getType() ) self.setDescrShort( obj.getDescrShort() ) self.setDescription( obj.getDescription() ) self.setOrigin( obj.getOrigin() ) self.setVersion( obj.getVersion() ) # copy instances and definitions self.parameters = ParameterCollection( obj.parameters ) self.module_definitions = DefinitionsPool( self, obj.module_definitions ) self.step_instances = InstancesPool( self, obj.step_instances ) self.step_definitions = DefinitionsPool( self, obj.step_definitions )
def __init__(self, name, obj=None, parent=None):
AttributeCollection.__init__(self)
self.parent = None
if obj == None:
self.parameters = ParameterCollection()
elif isinstance(obj, StepInstance) or isinstance(obj, StepDefinition):
if name == None:
self.setName(obj.getName())
else:
self.setName(name)
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.parameters = ParameterCollection(obj.parameters)
elif (obj == None) or isinstance(obj, ParameterCollection):
# set attributes
self.setName(name)
self.setType("")
self.setDescrShort("")
self.parameters = ParameterCollection(obj)
elif obj != None:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
self.step_commons = {}
self.stepStatus = S_OK()
def __init__(self, obj=None, name=None):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__(self)
if (obj == None) or isinstance(obj, ParameterCollection):
self.setName('notgiven')
self.setType('')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, Workflow):
self.fromWorkflow(obj)
elif isinstance(obj, str):
self.parameters = ParameterCollection(None)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
# if obj is an XML string
if obj.startswith('<'):
fromXMLString(obj, self)
else:
fromXMLFile(obj, self)
elif obj != None:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if name:
self.setName(name)
self.workflow_commons = {}
self.workflowStatus = S_OK()
def __init__( self, obj = None, name = None ):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__( self )
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setName( 'notgiven' )
self.setType( '' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, Workflow ):
self.fromWorkflow( obj )
elif isinstance( obj, str ):
self.parameters = ParameterCollection( None )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
# if obj is an XML string
if obj.startswith( '<' ):
fromXMLString( obj, self )
else:
fromXMLFile( obj, self )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if name :
self.setName( name )
self.workflow_commons = {}
self.workflowStatus = S_OK()
def __init__(self, step_type=None, obj=None, parent=None):
AttributeCollection.__init__(self)
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if (obj == None) or isinstance(obj, ParameterCollection):
self.setType('notgiven')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj) # creating copy
self.module_instances = InstancesPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, StepDefinition):
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(self, obj.parameters)
self.module_instances = InstancesPool(self, obj.module_instances)
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool(
self.obj.module_definitions)
else:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if step_type:
self.setType(step_type)
def __init__( self, name, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.instance_obj = None # used for the interpretation only
self.parent = parent
if obj == None:
self.parameters = ParameterCollection()
elif isinstance( obj, ModuleInstance ) or isinstance( obj, ModuleDefinition ):
if name == None:
self.setName( obj.getName() )
else:
self.setName( name )
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.parameters = ParameterCollection( obj.parameters )
elif isinstance( obj, ParameterCollection ):
# set attributes
self.setName( name )
self.setType( "" )
self.setDescrShort( "" )
self.parameters = ParameterCollection( obj )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
class StepDefinition( AttributeCollection ):
def __init__( self, step_type = None, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setType( 'notgiven' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj ) # creating copy
self.module_instances = InstancesPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, StepDefinition ):
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
# copy instances and definitions
self.parameters = ParameterCollection( self, obj.parameters )
self.module_instances = InstancesPool( self, obj.module_instances )
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool( self. obj.module_definitions )
else:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if step_type :
self.setType( step_type )
def __str__( self ):
ret = str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
if self.module_definitions != None:
ret = ret + str( self.module_definitions )
else:
ret = ret + 'Module definitions shared in Workflow\n'
ret = ret + str( self.module_instances )
return ret
def toXML( self ):
ret = '<StepDefinition>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
if self.module_definitions != None:
ret = ret + self.module_definitions.toXML()
ret = ret + self.module_instances.toXML()
ret = ret + '</StepDefinition>\n'
return ret
def toXMLFile( self, outFile ):
if os.path.exists( outFile ):
os.remove( outFile )
xmlfile = open( outFile, 'w' )
xmlfile.write( self.toXML() )
xmlfile.close()
def addModule( self, module ):
# KGG We need to add code to update existing modules
if self.module_definitions == None:
self.parent.module_definitions.append( module )
else:
self.module_definitions.append( module )
return module
def createModuleInstance( self, module_type, name ):
''' Creates module instance of type 'type' with the name 'name'
'''
if self.module_definitions[module_type]:
mi = ModuleInstance( name, self.module_definitions[module_type] )
self.module_instances.append( mi )
return mi
else:
raise KeyError( 'Can not find ModuleDefinition ' + module_type + ' to create ModuleInstrance ' + name )
def removeModuleInstance( self, name ):
''' Remove module instance specified by its name
'''
self.module_instances.delete( name )
def compare( self, s ):
''' Custom Step comparison operation
'''
ret = AttributeCollection.compare( self, s ) and self.module_instances.compare( s )
if self.module_definitions.getOwner() == self:
ret = ret and self.module_definitions.compare( s )
return ret
def updateParents( self, parent ):
'''
'''
#AttributeCollection.updateParents( self, parent )
self.module_instances.updateParents( self )
if( self.module_definitions != None ):
self.module_definitions.updateParents( self )
def createCode( self ):
''' Create Step code
'''
str_ = 'class ' + self.getType() + ':\n'
str_ = str_ + indent( 1 ) + 'def execute(self):\n'
str_ = str_ + self.module_instances.createCode()
str_ = str_ + indent( 2 ) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode( 2, 'self' )
str_ += '\n'
return str_
class StepInstance( AttributeCollection ):
def __init__( self, name, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.parent = None
if obj == None:
self.parameters = ParameterCollection()
elif isinstance( obj, StepInstance ) or isinstance( obj, StepDefinition ):
if name == None:
self.setName( obj.getName() )
else:
self.setName( name )
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.parameters = ParameterCollection( obj.parameters )
elif ( obj == None ) or isinstance( obj, ParameterCollection ):
# set attributes
self.setName( name )
self.setType( "" )
self.setDescrShort( "" )
self.parameters = ParameterCollection( obj )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
self.step_commons = {}
self.stepStatus = S_OK()
def resolveGlobalVars( self, step_definitions, wf_parameters ):
''' Resolve parameter values defined in the @{<variable>} form
'''
self.parameters.resolveGlobalVars( wf_parameters )
module_instance_number = 0
for inst in step_definitions[self.getType()].module_instances:
module_instance_number = module_instance_number + 1
if not inst.parameters.find( "MODULE_NUMBER" ):
inst.parameters.append( Parameter( "MODULE_NUMBER", "%s" % module_instance_number, "string", "", "",
True, False, "ModuleInstance number within the Step" ) )
if not inst.parameters.find( "MODULE_INSTANCE_NAME" ):
inst.parameters.append( Parameter( "MODULE_INSTANCE_NAME", inst.getName(), "string", "", "",
True, False, "Name of the ModuleInstance within the Step" ) )
if not inst.parameters.find( "MODULE_DEFINITION_NAME" ):
inst.parameters.append( Parameter( "MODULE_DEFINITION_NAME", inst.getType(), "string", "", "",
True, False, "Type of the ModuleInstance within the Step" ) )
if not inst.parameters.find( "JOB_ID" ):
inst.parameters.append( Parameter( "JOB_ID", "", "string", "self", "JOB_ID",
True, False, "Job ID within a Production as a string" ) )
if not inst.parameters.find( "PRODUCTION_ID" ):
inst.parameters.append( Parameter( "PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID",
True, False, "Production ID as a string" ) )
if not inst.parameters.find( "STEP_NUMBER" ):
inst.parameters.append( Parameter( "STEP_NUMBER", "", "string", "self", "STEP_NUMBER",
True, False, "Step instance number within the Workflow" ) )
if not inst.parameters.find( "STEP_ID" ):
inst.parameters.append( Parameter( "STEP_ID", "", "string", "self", "STEP_NUMBER",
True, False, "Step ID within the Workflow" ) )
inst.resolveGlobalVars( wf_parameters, self.parameters )
def createCode( self, ind = 2 ):
''' Create the Step code
'''
str_ = indent( ind ) + self.getName() + ' = ' + self.getType() + '()\n'
str_ = str_ + self.parameters.createParametersCode( ind, self.getName() )
str_ = str_ + indent( ind ) + self.getName() + '.execute()\n\n'
return str_
def __str__( self ):
''' Step string representation
'''
return str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
def toXML( self ):
''' Generate the Step XML representation
'''
ret = '<StepInstance>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
ret = ret + '</StepInstance>\n'
return ret
def setWorkflowCommons( self, wf ):
''' Add reference to the collection of the common tools
'''
self.workflow_commons = wf
def execute( self, step_exec_attr, definitions ):
''' Step execution method. step_exec_attr is array to hold parameters belong to this Step,
filled above in the workflow
'''
print 'Executing StepInstance', self.getName(), 'of type', self.getType(), definitions.keys()
# Report the Application state if the coresponding tool is supplied
if self.workflow_commons.has_key( 'JobReport' ):
if self.parent.workflowStatus['OK']:
result = self.workflow_commons['JobReport'].setApplicationStatus( 'Executing ' + self.getName() )
# Prepare Step statistics evaluation
self.step_commons['StartTime'] = time.time()
self.step_commons['StartStats'] = os.times()
step_def = definitions[self.getType()]
step_exec_modules = {}
error_message = ''
for mod_inst in step_def.module_instances:
mod_inst_name = mod_inst.getName()
mod_inst_type = mod_inst.getType()
# print "StepInstance creating module instance ", mod_inst_name, " of type", mod_inst.getType()
step_exec_modules[mod_inst_name] = \
step_def.parent.module_definitions[mod_inst_type].main_class_obj() # creating instance
# Resolve all the linked parameter values
for parameter in mod_inst.parameters:
if parameter.preExecute():
# print '>>>> Input', parameter
if parameter.isLinked():
# print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
setattr( step_exec_modules[mod_inst_name],
parameter.getName(),
step_exec_attr[parameter.getLinkedParameter()] )
else:
setattr( step_exec_modules[mod_inst_name],
parameter.getName(),
getattr( step_exec_modules[parameter.getLinkedModule()],
parameter.getLinkedParameter() ) )
else:
# print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
setattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() )
# print 'Step Input Parameter:', parameter.getName(), getattr( step_exec_modules[mod_inst_name], parameter.getName() )
# Set reference to the workflow and step common tools
setattr( step_exec_modules[mod_inst_name], 'workflow_commons', self.parent.workflow_commons )
setattr( step_exec_modules[mod_inst_name], 'step_commons', self.step_commons )
setattr( step_exec_modules[mod_inst_name], 'stepStatus', self.stepStatus )
setattr( step_exec_modules[mod_inst_name], 'workflowStatus', self.parent.workflowStatus )
try:
result = step_exec_modules[mod_inst_name].execute()
if not result['OK']:
if self.stepStatus['OK']:
error_message = result['Message']
if self.workflow_commons.has_key( 'JobReport' ):
if self.parent.workflowStatus['OK']:
resultStatus = self.workflow_commons['JobReport'].setApplicationStatus( error_message )
self.stepStatus = S_ERROR( result['Message'] )
else:
for parameter in mod_inst.parameters:
if parameter.isOutput():
# print '<<<< Output', parameter
if parameter.isLinked():
# print "ModuleInstance self ." + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# this is not supposed to happen
print "Warning! Module OUTPUT attribute", parameter.getName(),
print "refer to the attribute of the same module", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName() )
step_exec_attr[parameter.getName()] = getattr( step_exec_modules[mod_inst_name], parameter.getLinkedParameter(), parameter.getValue() )
# print " OUT", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() )
else:
# print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], parameter.getLinkedParameter()
step_exec_attr[parameter.getName()] = \
getattr( step_exec_modules[parameter.getLinkedModule()],
parameter.getLinkedParameter() )
else:
# This also does not make sense - we can give a warning
print "Warning! Module OUTPUT attribute ", parameter.getName(),
print "assigned constant", parameter.getValue()
# print "StepInstance self." + parameter.getName(), '=', parameter.getValue()
step_exec_attr[parameter.getName()] = parameter.getValue()
# print 'Module Output Parameter:', parameter.getName(), step_exec_attr[parameter.getName()]
# Get output values to the step_commons dictionary
for key in result.keys():
if key != "OK":
if key != "Value":
self.step_commons[key] = result[key]
elif type( result['Value'] ) == types.DictType:
for vkey in result['Value'].keys():
self.step_commons[vkey] = result['Value'][vkey]
except Exception, x:
print "Exception while module execution"
print "Module", mod_inst_name, mod_inst.getType()
print str( x )
exc = sys.exc_info()
exc_type = exc[0]
value = exc[1]
print "== EXCEPTION ==\n%s: %s\n\n%s===============" % (
exc_type,
value,
"\n".join( traceback.format_tb( exc[2] ) ) )
print "Step status: ", self.stepStatus
print "Workflow status: ", self.parent.workflowStatus
if self.stepStatus['OK']:
# This is the error that caused the workflow disruption
# report it to the WMS
error_message = 'Exception while %s module execution: %s' % ( mod_inst_name, str( x ) )
if self.workflow_commons.has_key( 'JobReport' ):
if self.parent.workflowStatus['OK']:
result = self.workflow_commons['JobReport'].setApplicationStatus( 'Exception in %s module' % mod_inst_name )
self.stepStatus = S_ERROR( error_message )
# now we need to copy output values to the STEP!!! parameters
for st_parameter in self.parameters:
if st_parameter.isOutput():
# print '<< Output', st_parameter
if st_parameter.isLinked():
# print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getLinkedModule() + '.' + st_parameter.getLinkedParameter()
if st_parameter.getLinkedModule() == 'self':
# this is not supposed to happen
print "Warning! Step OUTPUT attribute", st_parameter.getName(),
print "refer to the attribute of the same step", st_parameter.getLinkedParameter(), step_exec_attr[st_parameter.getLinkedParameter()]
step_exec_attr[st_parameter.getName()] = step_exec_attr[st_parameter.getLinkedParameter()]
else:
# print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter()
step_exec_attr[st_parameter.getName()] = \
getattr( step_exec_modules[st_parameter.getLinkedModule()],
st_parameter.getLinkedParameter() )
setattr( self, st_parameter.getName(), step_exec_attr[st_parameter.getName()] )
else:
# This also does not make sense - we can give a warning
print "Warning! Step OUTPUT attribute ", st_parameter.getName(),
print "assigned constant", st_parameter.getValue()
# print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getValue()
step_exec_attr[st_parameter.getName()] = st_parameter.getValue()
print 'Step Output', st_parameter.getName(), '=', step_exec_attr[st_parameter.getName()]
# Return the result of the first failed module or S_OK
if not self.stepStatus['OK']:
return S_ERROR( error_message )
else:
return S_OK( result['Value'] )
class Workflow(AttributeCollection):
def __init__(self, obj=None, name=None):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__(self)
if (obj == None) or isinstance(obj, ParameterCollection):
self.setName('notgiven')
self.setType('')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, Workflow):
self.fromWorkflow(obj)
elif isinstance(obj, str):
self.parameters = ParameterCollection(None)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
# if obj is an XML string
if obj.startswith('<'):
fromXMLString(obj, self)
else:
fromXMLFile(obj, self)
elif obj != None:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if name:
self.setName(name)
self.workflow_commons = {}
self.workflowStatus = S_OK()
def fromWorkflow(self, obj):
self.setName(obj.getName())
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(obj.parameters)
self.module_definitions = DefinitionsPool(self, obj.module_definitions)
self.step_instances = InstancesPool(self, obj.step_instances)
self.step_definitions = DefinitionsPool(self, obj.step_definitions)
def __str__(self):
'''Creates a string representation of itself
'''
ret = str(self.getName()) + ':\n' + AttributeCollection.__str__(
self) + self.parameters.__str__()
ret = ret + str(self.step_definitions)
ret = ret + str(self.step_instances)
ret = ret + str(self.module_definitions)
return ret
def toXML(self):
'''Creates an XML representation of itself
'''
# THIS is very important that Definitions should be written before instances
ret = '<Workflow>\n'
ret = ret + AttributeCollection.toXML(self)
ret = ret + self.parameters.toXML()
ret = ret + self.module_definitions.toXML()
ret = ret + self.step_definitions.toXML()
ret = ret + self.step_instances.toXML()
ret = ret + '</Workflow>\n'
return ret
def toXMLFile(self, outFile):
if os.path.exists(outFile):
os.remove(outFile)
xmlfile = open(outFile, 'w')
xmlfile.write(self.toXML())
xmlfile.close()
def addTool(self, name, tool):
''' Add an object that will be available in all the modules to perform some operations.
For example, a state reporting facility.
'''
self.workflow_commons[name] = tool
def addStep(self, step):
# this is a VERY important piece of code
# we have to join all Modules definition from all added steps in the single dictionary
# and we have to share this dictionary between all included steps
# we also have to check versions of the modules and instances
for type_ in step.module_definitions.keys():
#if self.module_definitions.has_key(type):
# we have the same ModuleDefinition in 2 places
# we need to find way to synchronise it
#print "Workflow:addStep - we need to write ModuleDefinitions synchronisation code"
#else:
# new module - just append it
if not self.module_definitions.has_key(type_):
self.module_definitions.append(step.module_definitions[type_])
self.step_definitions.append(step)
del step.module_definitions # we need to clean up all unwanted definitions
step.module_definitions = None
return step
def addModule(self, module):
# KGG We need to add code to update existing modules
self.module_definitions.append(module)
return module
def createStepInstance(self, type_, name):
''' Creates step instance of type 'type' with the name 'name'
'''
if self.step_definitions.has_key(type_):
stepi = StepInstance(name, self.step_definitions[type_])
self.step_instances.append(stepi)
return stepi
else:
raise KeyError('Can not find StepDefinition ' + type_ +
' to create StepInstrance ' + name)
def removeStepInstance(self, name):
self.instances[name].setParents(None)
self.instances.delete(name)
def updateParents(self):
self.module_definitions.updateParents(self)
self.step_instances.updateParents(self)
self.step_definitions.updateParents(self)
def resolveGlobalVars(self):
''' This method will create global parameter list and then will resolve all instances of @{VARNAME}
Be aware that parameters of that type are GLOBAL!!! are string and can not be dynamically change
The scope: the resolution of that parameters apply from lower to upper object, for example if
parameter use in module, then it checks module, then step, then workflow
Comment: If varible linked it should not be used in a global list'''
# reenforced global parameters on the level of Workflow
if not self.parameters.find("PRODUCTION_ID"):
self.parameters.append(
Parameter(
"PRODUCTION_ID", "00000000", "string", "", "", True, False,
"Transformation ID taken from the ProductionManager"))
if not self.parameters.find("JOB_ID"):
self.parameters.append(
Parameter(
"JOB_ID", "00000000", "string", "", "", True, False,
"Job ID within Tranformationtaken from the ProductionManager"
))
self.parameters.resolveGlobalVars()
step_instance_number = 0
for inst in self.step_instances:
# for each step instance we can define STEP_NUMBER
step_instance_number = step_instance_number + 1
if not inst.parameters.find("STEP_NUMBER"):
inst.parameters.append(
Parameter(
"STEP_NUMBER", "%s" % step_instance_number, "string",
"", "", True, False,
"Number of the StepInstance within the Workflow"))
if not inst.parameters.find("STEP_ID"):
prod_ID = self.parameters.find("PRODUCTION_ID").getValue()
job_ID = self.parameters.find("JOB_ID").getValue()
inst.parameters.append(
Parameter(
"STEP_ID",
"%s_%s_%d" % (prod_ID, job_ID, step_instance_number),
"string", "", "", True, False, "Step instance ID"))
if not inst.parameters.find("STEP_INSTANCE_NAME"):
inst.parameters.append(
Parameter("STEP_INSTANCE_NAME", inst.getName(), "string",
"", "", True, False,
"Name of the StepInstance within the Workflow"))
if not inst.parameters.find("STEP_DEFINITION_NAME"):
inst.parameters.append(
Parameter("STEP_DEFINITION_NAME", inst.getType(), "string",
"", "", True, False,
"Type of the StepInstance within the Workflow"))
if not inst.parameters.find("JOB_ID"):
inst.parameters.append(
Parameter("JOB_ID", "", "string", "self", "JOB_ID", True,
False,
"Type of the StepInstance within the Workflow"))
if not inst.parameters.find("PRODUCTION_ID"):
inst.parameters.append(
Parameter("PRODUCTION_ID", "", "string", "self",
"PRODUCTION_ID", True, False,
"Type of the StepInstance within the Workflow"))
inst.resolveGlobalVars(self.step_definitions, self.parameters)
def createCode(self, combine_steps=False):
self.resolveGlobalVars()
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent(1) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent(2) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode(2, 'self')
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode(0, 'j')
str_ = str_ + 'j.execute()'
return str_
def showCode(self, combine_steps=False):
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent(1) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent(2) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode(2, 'self')
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode(0, 'j')
str_ = str_ + 'j.execute()'
return str_
def execute(self):
self.resolveGlobalVars()
# define workflow attributes
wf_exec_attr = {
} # dictianary with the WF attributes, used to resolve links to self.attrname
for wf_parameter in self.parameters:
# parameters shall see objects in the current scope order to resolve links
if wf_parameter.preExecute(): # for parm which not just outputs
# print 'Input', wf_parameter
if wf_parameter.isLinked():
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Job attribute ", wf_parameter.getName(
), "refers to the attribute of the same workflow", wf_parameter.getLinkedParameter(
)
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[
wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[
wf_parameter.getLinkedModule()][
wf_parameter.getLinkedParameter()]
else:
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getValue()
wf_exec_attr[
wf_parameter.getName()] = wf_parameter.getValue()
# Put all the workflow parameters into the workflow_commons dictionary
self.workflow_commons[
wf_parameter.getName()] = wf_parameter.getValue()
self.module_definitions.loadCode(
) # loading Module classes into current python scope
#wf_exec_steps will be dictionary of dictionaries [step instance name][parameter name]
# used as dictionary of step instances to carry parameters
wf_exec_steps = {}
# print 'Executing Workflow',self.getType()
error_message = ''
step_result = ''
for step_inst in self.step_instances:
step_inst_name = step_inst.getName()
wf_exec_steps[step_inst_name] = {}
# step_inst_type = step_inst.getType()
# print "WorkflowInstance creating Step instance ",step_inst_name," of type", step_inst_type
for parameter in step_inst.parameters:
if parameter.preExecute():
# print '>> Input', parameter
if parameter.isLinked():
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
wf_exec_steps[step_inst_name][parameter.getName(
)] = wf_exec_attr[parameter.getLinkedParameter()]
else:
# print wf_exec_steps[parameter.getLinkedModule()].keys()
wf_exec_steps[step_inst_name][parameter.getName(
)] = wf_exec_steps[parameter.getLinkedModule()][
parameter.getLinkedParameter()]
else:
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
wf_exec_steps[step_inst_name][
parameter.getName()] = parameter.getValue()
# In the step_commons all parameters are added, both Input and Output ones.
step_inst.step_commons[
parameter.getName()] = parameter.getValue()
resolveVariables(wf_exec_steps[step_inst_name])
# Set proper values for all Input Parameters
for key, value in wf_exec_steps[step_inst_name].items():
step_inst.step_commons[key] = value
step_inst.setParent(self)
step_inst.setWorkflowCommons(self.workflow_commons)
result = step_inst.execute(wf_exec_steps[step_inst_name],
self.step_definitions)
if not result['OK']:
if self.workflowStatus['OK']:
error_message = result['Message']
self.workflowStatus = S_ERROR(result['Message'])
if result.has_key('Value'):
step_result = result['Value']
# now we need to copy output values to the STEP!!! parameters
#print "WorkflowInstance output assignment"
for wf_parameter in self.parameters:
if wf_parameter.isOutput():
if wf_parameter.isLinked():
# print "WorkflowInstance self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Workflow OUTPUT attribute ", wf_parameter.getName(
), "refer on the attribute of the same workflow", wf_parameter.getLinkedParameter(
)
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[
wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_steps[
wf_parameter.getLinkedModule()][
wf_parameter.getLinkedParameter()]
else:
# it is also does not make sense - we can produce warning
print "Warning! Workflow OUTPUT attribute", wf_parameter.getName(
), "assigned constant", wf_parameter.getValue()
# print "WorkflowInstance self."+ wf_parameter.getName(),'=',wf_parameter.getValue()
wf_exec_attr[
wf_parameter.getName()] = wf_parameter.getValue()
setattr(self, wf_parameter.getName(),
wf_exec_attr[wf_parameter.getName()])
# Return the result of the first failed step or S_OK
if not self.workflowStatus['OK']:
return S_ERROR(error_message)
else:
return S_OK(step_result)
class Workflow( AttributeCollection ):
def __init__( self, obj = None, name = None ):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__( self )
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setName( 'notgiven' )
self.setType( '' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, Workflow ):
self.fromWorkflow( obj )
elif isinstance( obj, str ):
self.parameters = ParameterCollection( None )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
# if obj is an XML string
if obj.startswith( '<' ):
fromXMLString( obj, self )
else:
fromXMLFile( obj, self )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if name :
self.setName( name )
self.workflow_commons = {}
self.workflowStatus = S_OK()
def fromWorkflow( self, obj ):
self.setName( obj.getName() )
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
# copy instances and definitions
self.parameters = ParameterCollection( obj.parameters )
self.module_definitions = DefinitionsPool( self, obj.module_definitions )
self.step_instances = InstancesPool( self, obj.step_instances )
self.step_definitions = DefinitionsPool( self, obj.step_definitions )
def __str__( self ):
'''Creates a string representation of itself
'''
ret = str( self.getName() ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
ret = ret + str( self.step_definitions )
ret = ret + str( self.step_instances )
ret = ret + str( self.module_definitions )
return ret
def toXML( self ):
'''Creates an XML representation of itself
'''
# THIS is very important that Definitions should be written before instances
ret = '<Workflow>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
ret = ret + self.module_definitions.toXML()
ret = ret + self.step_definitions.toXML()
ret = ret + self.step_instances.toXML()
ret = ret + '</Workflow>\n'
return ret
def toXMLFile( self, outFile ):
if os.path.exists( outFile ):
os.remove( outFile )
xmlfile = open( outFile, 'w' )
xmlfile.write( self.toXML() )
xmlfile.close()
def addTool( self, name, tool ):
''' Add an object that will be available in all the modules to perform some operations.
For example, a state reporting facility.
'''
self.workflow_commons[name] = tool
def addStep( self, step ):
# this is a VERY important piece of code
# we have to join all Modules definition from all added steps in the single dictionary
# and we have to share this dictionary between all included steps
# we also have to check versions of the modules and instances
for type_ in step.module_definitions.keys():
#if self.module_definitions.has_key(type):
# we have the same ModuleDefinition in 2 places
# we need to find way to synchronise it
#print "Workflow:addStep - we need to write ModuleDefinitions synchronisation code"
#else:
# new module - just append it
if not self.module_definitions.has_key( type_ ):
self.module_definitions.append( step.module_definitions[type_] )
self.step_definitions.append( step )
del step.module_definitions # we need to clean up all unwanted definitions
step.module_definitions = None
return step
def addModule( self, module ):
# KGG We need to add code to update existing modules
self.module_definitions.append( module )
return module
def createStepInstance( self, type_, name ):
''' Creates step instance of type 'type' with the name 'name'
'''
if self.step_definitions.has_key( type_ ):
stepi = StepInstance( name, self.step_definitions[type_] )
self.step_instances.append( stepi )
return stepi
else:
raise KeyError( 'Can not find StepDefinition ' + type_ + ' to create StepInstrance ' + name )
def removeStepInstance( self, name ):
self.instances[name].setParents( None )
self.instances.delete( name )
def updateParents( self ):
self.module_definitions.updateParents( self )
self.step_instances.updateParents( self )
self.step_definitions.updateParents( self )
def resolveGlobalVars( self ):
''' This method will create global parameter list and then will resolve all instances of @{VARNAME}
Be aware that parameters of that type are GLOBAL!!! are string and can not be dynamically change
The scope: the resolution of that parameters apply from lower to upper object, for example if
parameter use in module, then it checks module, then step, then workflow
Comment: If varible linked it should not be used in a global list'''
# reenforced global parameters on the level of Workflow
if not self.parameters.find( "PRODUCTION_ID" ):
self.parameters.append( Parameter( "PRODUCTION_ID", "00000000", "string", "", "", True, False, "Transformation ID taken from the ProductionManager" ) )
if not self.parameters.find( "JOB_ID" ):
self.parameters.append( Parameter( "JOB_ID", "00000000", "string", "", "", True, False, "Job ID within Tranformationtaken from the ProductionManager" ) )
self.parameters.resolveGlobalVars()
step_instance_number = 0
for inst in self.step_instances:
# for each step instance we can define STEP_NUMBER
step_instance_number = step_instance_number + 1
if not inst.parameters.find( "STEP_NUMBER" ):
inst.parameters.append( Parameter( "STEP_NUMBER", "%s" % step_instance_number, "string", "", "", True, False, "Number of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "STEP_ID" ):
prod_ID = self.parameters.find( "PRODUCTION_ID" ).getValue()
job_ID = self.parameters.find( "JOB_ID" ).getValue()
inst.parameters.append( Parameter( "STEP_ID", "%s_%s_%d" % ( prod_ID, job_ID, step_instance_number ), "string", "", "", True, False, "Step instance ID" ) )
if not inst.parameters.find( "STEP_INSTANCE_NAME" ):
inst.parameters.append( Parameter( "STEP_INSTANCE_NAME", inst.getName(), "string", "", "", True, False, "Name of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "STEP_DEFINITION_NAME" ):
inst.parameters.append( Parameter( "STEP_DEFINITION_NAME", inst.getType(), "string", "", "", True, False, "Type of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "JOB_ID" ):
inst.parameters.append( Parameter( "JOB_ID", "", "string", "self", "JOB_ID", True, False, "Type of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "PRODUCTION_ID" ):
inst.parameters.append( Parameter( "PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID", True, False, "Type of the StepInstance within the Workflow" ) )
inst.resolveGlobalVars( self.step_definitions, self.parameters )
def createCode( self, combine_steps = False ):
self.resolveGlobalVars()
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent( 1 ) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent( 2 ) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode( 2, 'self' )
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode( 0, 'j' )
str_ = str_ + 'j.execute()'
return str_
def showCode( self, combine_steps = False ):
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent( 1 ) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent( 2 ) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode( 2, 'self' )
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode( 0, 'j' )
str_ = str_ + 'j.execute()'
return str_
def execute( self ):
self.resolveGlobalVars()
# define workflow attributes
wf_exec_attr = {} # dictianary with the WF attributes, used to resolve links to self.attrname
for wf_parameter in self.parameters:
# parameters shall see objects in the current scope order to resolve links
if wf_parameter.preExecute(): # for parm which not just outputs
# print 'Input', wf_parameter
if wf_parameter.isLinked():
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Job attribute ", wf_parameter.getName(), "refers to the attribute of the same workflow", wf_parameter.getLinkedParameter()
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedModule()][wf_parameter.getLinkedParameter()]
else:
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getValue()
wf_exec_attr[wf_parameter.getName()] = wf_parameter.getValue()
# Put all the workflow parameters into the workflow_commons dictionary
self.workflow_commons[wf_parameter.getName()] = wf_parameter.getValue()
self.module_definitions.loadCode() # loading Module classes into current python scope
#wf_exec_steps will be dictionary of dictionaries [step instance name][parameter name]
# used as dictionary of step instances to carry parameters
wf_exec_steps = {}
# print 'Executing Workflow',self.getType()
error_message = ''
step_result = ''
for step_inst in self.step_instances:
step_inst_name = step_inst.getName()
wf_exec_steps[step_inst_name] = {}
# step_inst_type = step_inst.getType()
# print "WorkflowInstance creating Step instance ",step_inst_name," of type", step_inst_type
for parameter in step_inst.parameters:
if parameter.preExecute():
# print '>> Input', parameter
if parameter.isLinked():
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
wf_exec_steps[step_inst_name][parameter.getName()] = wf_exec_attr[parameter.getLinkedParameter()]
else:
# print wf_exec_steps[parameter.getLinkedModule()].keys()
wf_exec_steps[step_inst_name][parameter.getName()] = wf_exec_steps[parameter.getLinkedModule()][parameter.getLinkedParameter()]
else:
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
wf_exec_steps[step_inst_name][parameter.getName()] = parameter.getValue()
# In the step_commons all parameters are added, both Input and Output ones.
step_inst.step_commons[parameter.getName()] = parameter.getValue()
resolveVariables( wf_exec_steps[step_inst_name] )
# Set proper values for all Input Parameters
for key, value in wf_exec_steps[step_inst_name].items():
step_inst.step_commons[key] = value
step_inst.setParent( self )
step_inst.setWorkflowCommons( self.workflow_commons )
result = step_inst.execute( wf_exec_steps[step_inst_name], self.step_definitions )
if not result['OK']:
if self.workflowStatus['OK']:
error_message = result['Message']
self.workflowStatus = S_ERROR( result['Message'] )
if result.has_key( 'Value' ):
step_result = result['Value']
# now we need to copy output values to the STEP!!! parameters
#print "WorkflowInstance output assignment"
for wf_parameter in self.parameters:
if wf_parameter.isOutput():
if wf_parameter.isLinked():
# print "WorkflowInstance self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Workflow OUTPUT attribute ", wf_parameter.getName(), "refer on the attribute of the same workflow", wf_parameter.getLinkedParameter()
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_steps[wf_parameter.getLinkedModule()][wf_parameter.getLinkedParameter()]
else:
# it is also does not make sense - we can produce warning
print "Warning! Workflow OUTPUT attribute", wf_parameter.getName(), "assigned constant", wf_parameter.getValue()
# print "WorkflowInstance self."+ wf_parameter.getName(),'=',wf_parameter.getValue()
wf_exec_attr[wf_parameter.getName()] = wf_parameter.getValue()
setattr( self, wf_parameter.getName(), wf_exec_attr[wf_parameter.getName()] )
# Return the result of the first failed step or S_OK
if not self.workflowStatus['OK']:
return S_ERROR( error_message )
else:
return S_OK( step_result )
class StepDefinition(AttributeCollection):
def __init__(self, step_type=None, obj=None, parent=None):
AttributeCollection.__init__(self)
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if (obj == None) or isinstance(obj, ParameterCollection):
self.setType('notgiven')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj) # creating copy
self.module_instances = InstancesPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, StepDefinition):
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(self, obj.parameters)
self.module_instances = InstancesPool(self, obj.module_instances)
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool(
self.obj.module_definitions)
else:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if step_type:
self.setType(step_type)
def __str__(self):
ret = str(type(self)) + ':\n' + AttributeCollection.__str__(
self) + self.parameters.__str__()
if self.module_definitions != None:
ret = ret + str(self.module_definitions)
else:
ret = ret + 'Module definitions shared in Workflow\n'
ret = ret + str(self.module_instances)
return ret
def toXML(self):
ret = '<StepDefinition>\n'
ret = ret + AttributeCollection.toXML(self)
ret = ret + self.parameters.toXML()
if self.module_definitions != None:
ret = ret + self.module_definitions.toXML()
ret = ret + self.module_instances.toXML()
ret = ret + '</StepDefinition>\n'
return ret
def toXMLFile(self, outFile):
if os.path.exists(outFile):
os.remove(outFile)
xmlfile = open(outFile, 'w')
xmlfile.write(self.toXML())
xmlfile.close()
def addModule(self, module):
# KGG We need to add code to update existing modules
if self.module_definitions == None:
self.parent.module_definitions.append(module)
else:
self.module_definitions.append(module)
return module
def createModuleInstance(self, module_type, name):
''' Creates module instance of type 'type' with the name 'name'
'''
if self.module_definitions[module_type]:
mi = ModuleInstance(name, self.module_definitions[module_type])
self.module_instances.append(mi)
return mi
else:
raise KeyError('Can not find ModuleDefinition ' + module_type +
' to create ModuleInstrance ' + name)
def removeModuleInstance(self, name):
''' Remove module instance specified by its name
'''
self.module_instances.delete(name)
def compare(self, s):
''' Custom Step comparison operation
'''
ret = AttributeCollection.compare(
self, s) and self.module_instances.compare(s)
if self.module_definitions.getOwner() == self:
ret = ret and self.module_definitions.compare(s)
return ret
def updateParents(self, parent):
'''
'''
#AttributeCollection.updateParents( self, parent )
self.module_instances.updateParents(self)
if (self.module_definitions != None):
self.module_definitions.updateParents(self)
def createCode(self):
''' Create Step code
'''
str_ = 'class ' + self.getType() + ':\n'
str_ = str_ + indent(1) + 'def execute(self):\n'
str_ = str_ + self.module_instances.createCode()
str_ = str_ + indent(2) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode(2, 'self')
str_ += '\n'
return str_
class StepInstance(AttributeCollection):
def __init__(self, name, obj=None, parent=None):
AttributeCollection.__init__(self)
self.parent = None
if obj == None:
self.parameters = ParameterCollection()
elif isinstance(obj, StepInstance) or isinstance(obj, StepDefinition):
if name == None:
self.setName(obj.getName())
else:
self.setName(name)
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.parameters = ParameterCollection(obj.parameters)
elif (obj == None) or isinstance(obj, ParameterCollection):
# set attributes
self.setName(name)
self.setType("")
self.setDescrShort("")
self.parameters = ParameterCollection(obj)
elif obj != None:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
self.step_commons = {}
self.stepStatus = S_OK()
def resolveGlobalVars(self, step_definitions, wf_parameters):
''' Resolve parameter values defined in the @{<variable>} form
'''
self.parameters.resolveGlobalVars(wf_parameters)
module_instance_number = 0
for inst in step_definitions[self.getType()].module_instances:
module_instance_number = module_instance_number + 1
if not inst.parameters.find("MODULE_NUMBER"):
inst.parameters.append(
Parameter("MODULE_NUMBER", "%s" % module_instance_number,
"string", "", "", True, False,
"ModuleInstance number within the Step"))
if not inst.parameters.find("MODULE_INSTANCE_NAME"):
inst.parameters.append(
Parameter("MODULE_INSTANCE_NAME", inst.getName(), "string",
"", "", True, False,
"Name of the ModuleInstance within the Step"))
if not inst.parameters.find("MODULE_DEFINITION_NAME"):
inst.parameters.append(
Parameter("MODULE_DEFINITION_NAME", inst.getType(),
"string", "", "", True, False,
"Type of the ModuleInstance within the Step"))
if not inst.parameters.find("JOB_ID"):
inst.parameters.append(
Parameter("JOB_ID", "", "string", "self", "JOB_ID", True,
False, "Job ID within a Production as a string"))
if not inst.parameters.find("PRODUCTION_ID"):
inst.parameters.append(
Parameter("PRODUCTION_ID", "", "string", "self",
"PRODUCTION_ID", True, False,
"Production ID as a string"))
if not inst.parameters.find("STEP_NUMBER"):
inst.parameters.append(
Parameter("STEP_NUMBER", "", "string", "self",
"STEP_NUMBER", True, False,
"Step instance number within the Workflow"))
if not inst.parameters.find("STEP_ID"):
inst.parameters.append(
Parameter("STEP_ID", "", "string", "self", "STEP_NUMBER",
True, False, "Step ID within the Workflow"))
inst.resolveGlobalVars(wf_parameters, self.parameters)
def createCode(self, ind=2):
''' Create the Step code
'''
str_ = indent(ind) + self.getName() + ' = ' + self.getType() + '()\n'
str_ = str_ + self.parameters.createParametersCode(ind, self.getName())
str_ = str_ + indent(ind) + self.getName() + '.execute()\n\n'
return str_
def __str__(self):
''' Step string representation
'''
return str(type(self)) + ':\n' + AttributeCollection.__str__(
self) + self.parameters.__str__()
def toXML(self):
''' Generate the Step XML representation
'''
ret = '<StepInstance>\n'
ret = ret + AttributeCollection.toXML(self)
ret = ret + self.parameters.toXML()
ret = ret + '</StepInstance>\n'
return ret
def setWorkflowCommons(self, wf):
''' Add reference to the collection of the common tools
'''
self.workflow_commons = wf
def execute(self, step_exec_attr, definitions):
''' Step execution method. step_exec_attr is array to hold parameters belong to this Step,
filled above in the workflow
'''
print 'Executing StepInstance', self.getName(
), 'of type', self.getType(), definitions.keys()
# Report the Application state if the coresponding tool is supplied
if self.workflow_commons.has_key('JobReport'):
if self.parent.workflowStatus['OK']:
result = self.workflow_commons[
'JobReport'].setApplicationStatus('Executing ' +
self.getName())
# Prepare Step statistics evaluation
self.step_commons['StartTime'] = time.time()
self.step_commons['StartStats'] = os.times()
step_def = definitions[self.getType()]
step_exec_modules = {}
error_message = ''
for mod_inst in step_def.module_instances:
mod_inst_name = mod_inst.getName()
mod_inst_type = mod_inst.getType()
# print "StepInstance creating module instance ", mod_inst_name, " of type", mod_inst.getType()
step_exec_modules[mod_inst_name] = \
step_def.parent.module_definitions[mod_inst_type].main_class_obj() # creating instance
# Resolve all the linked parameter values
for parameter in mod_inst.parameters:
if parameter.preExecute():
# print '>>>> Input', parameter
if parameter.isLinked():
# print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
setattr(
step_exec_modules[mod_inst_name],
parameter.getName(),
step_exec_attr[parameter.getLinkedParameter()])
else:
setattr(
step_exec_modules[mod_inst_name],
parameter.getName(),
getattr(
step_exec_modules[
parameter.getLinkedModule()],
parameter.getLinkedParameter()))
else:
# print ">>>> ModuleInstance", mod_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
setattr(step_exec_modules[mod_inst_name],
parameter.getName(), parameter.getValue())
# print 'Step Input Parameter:', parameter.getName(), getattr( step_exec_modules[mod_inst_name], parameter.getName() )
# Set reference to the workflow and step common tools
setattr(step_exec_modules[mod_inst_name], 'workflow_commons',
self.parent.workflow_commons)
setattr(step_exec_modules[mod_inst_name], 'step_commons',
self.step_commons)
setattr(step_exec_modules[mod_inst_name], 'stepStatus',
self.stepStatus)
setattr(step_exec_modules[mod_inst_name], 'workflowStatus',
self.parent.workflowStatus)
try:
result = step_exec_modules[mod_inst_name].execute()
if not result['OK']:
if self.stepStatus['OK']:
error_message = result['Message']
if self.workflow_commons.has_key('JobReport'):
if self.parent.workflowStatus['OK']:
resultStatus = self.workflow_commons[
'JobReport'].setApplicationStatus(
error_message)
self.stepStatus = S_ERROR(result['Message'])
else:
for parameter in mod_inst.parameters:
if parameter.isOutput():
# print '<<<< Output', parameter
if parameter.isLinked():
# print "ModuleInstance self ." + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# this is not supposed to happen
print "Warning! Module OUTPUT attribute", parameter.getName(
),
print "refer to the attribute of the same module", parameter.getLinkedParameter(
), '=', getattr(
step_exec_modules[mod_inst_name],
parameter.getName())
step_exec_attr[
parameter.getName()] = getattr(
step_exec_modules[mod_inst_name],
parameter.getLinkedParameter(),
parameter.getValue())
# print " OUT", parameter.getLinkedParameter(), '=', getattr( step_exec_modules[mod_inst_name], parameter.getName(), parameter.getValue() )
else:
# print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], parameter.getLinkedParameter()
step_exec_attr[parameter.getName()] = \
getattr( step_exec_modules[parameter.getLinkedModule()],
parameter.getLinkedParameter() )
else:
# This also does not make sense - we can give a warning
print "Warning! Module OUTPUT attribute ", parameter.getName(
),
print "assigned constant", parameter.getValue()
# print "StepInstance self." + parameter.getName(), '=', parameter.getValue()
step_exec_attr[parameter.getName(
)] = parameter.getValue()
# print 'Module Output Parameter:', parameter.getName(), step_exec_attr[parameter.getName()]
# Get output values to the step_commons dictionary
for key in result.keys():
if key != "OK":
if key != "Value":
self.step_commons[key] = result[key]
elif type(result['Value']) == types.DictType:
for vkey in result['Value'].keys():
self.step_commons[vkey] = result['Value'][
vkey]
except Exception, x:
print "Exception while module execution"
print "Module", mod_inst_name, mod_inst.getType()
print str(x)
exc = sys.exc_info()
exc_type = exc[0]
value = exc[1]
print "== EXCEPTION ==\n%s: %s\n\n%s===============" % (
exc_type, value, "\n".join(traceback.format_tb(exc[2])))
print "Step status: ", self.stepStatus
print "Workflow status: ", self.parent.workflowStatus
if self.stepStatus['OK']:
# This is the error that caused the workflow disruption
# report it to the WMS
error_message = 'Exception while %s module execution: %s' % (
mod_inst_name, str(x))
if self.workflow_commons.has_key('JobReport'):
if self.parent.workflowStatus['OK']:
result = self.workflow_commons[
'JobReport'].setApplicationStatus(
'Exception in %s module' % mod_inst_name)
self.stepStatus = S_ERROR(error_message)
# now we need to copy output values to the STEP!!! parameters
for st_parameter in self.parameters:
if st_parameter.isOutput():
# print '<< Output', st_parameter
if st_parameter.isLinked():
# print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getLinkedModule() + '.' + st_parameter.getLinkedParameter()
if st_parameter.getLinkedModule() == 'self':
# this is not supposed to happen
print "Warning! Step OUTPUT attribute", st_parameter.getName(
),
print "refer to the attribute of the same step", st_parameter.getLinkedParameter(
), step_exec_attr[st_parameter.getLinkedParameter()]
step_exec_attr[st_parameter.getName(
)] = step_exec_attr[st_parameter.getLinkedParameter()]
else:
# print 'Output step_exec_attr', st_parameter.getName(), step_exec_modules[st_parameter.getLinkedModule()], st_parameter.getLinkedParameter()
step_exec_attr[st_parameter.getName()] = \
getattr( step_exec_modules[st_parameter.getLinkedModule()],
st_parameter.getLinkedParameter() )
setattr(self, st_parameter.getName(),
step_exec_attr[st_parameter.getName()])
else:
# This also does not make sense - we can give a warning
print "Warning! Step OUTPUT attribute ", st_parameter.getName(
),
print "assigned constant", st_parameter.getValue()
# print "StepInstance self." + st_parameter.getName(), '=', st_parameter.getValue()
step_exec_attr[
st_parameter.getName()] = st_parameter.getValue()
print 'Step Output', st_parameter.getName(
), '=', step_exec_attr[st_parameter.getName()]
# Return the result of the first failed module or S_OK
if not self.stepStatus['OK']:
return S_ERROR(error_message)
else:
return S_OK(result['Value'])
class ModuleDefinition( AttributeCollection ):
def __init__( self, type_ = None, obj = None, parent = None ):
# we can create an object from another module
# or from the ParameterCollection
AttributeCollection.__init__( self )
self.main_class_obj = None # used for the interpretation only
# self.module_obj = None # used for the interpretation only
self.parent = parent
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setType( 'notgiven' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setRequired( '' )
self.setBody( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj ) # creating copy
elif isinstance( obj, ModuleDefinition ):
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setBody( obj.getBody() )
self.setRequired( obj.getRequired() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
self.parameters = ParameterCollection( obj.parameters )
else:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if type_ :
self.setType( type_ )
def createCode( self ):
return self.getBody() + '\n'
def __str__( self ):
return str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
def toXML( self ):
ret = '<ModuleDefinition>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
ret = ret + '</ModuleDefinition>\n'
return ret
def toXMLFile( self, outFile ):
if os.path.exists( outFile ):
os.remove( outFile )
xmlfile = open( outFile, 'w' )
xmlfile.write( self.toXML() )
xmlfile.close()
def loadCode( self ):
# print 'Loading code of the Module =', self.getType()
# version 1 - OLD sample
# ret = compile(self.getBody(),'<string>','exec')
# eval(ret)
# return ret #returning ref just in case we might need it
#
if len( self.getBody() ): # checking the size of the string
# version 2 - we assume that each self.body is a module oblect
# module = new.module(self.getType()) # create empty module object
# sys.modules[self.getType()] = module # add reference for the import operator
# exec self.getBody() in module.__dict__ # execute code itself
# self.module_obj = module # save pointer to this module
# if module.__dict__.has_key(self.getType()):
# self.main_class_obj = module.__dict__[self.getType()] # save class object
# version 3
# A.T. Use vars() function to inspect local objects instead of playing with
# fake modules. We assume that after the body execution there will be
# a class with name "self.getType()" defined in the local scope.
exec self.getBody()
if vars().has_key( self.getType() ):
self.main_class_obj = vars()[self.getType()] # save class object
else:
# it is possible to have this class in another module, we have to check for this
# but it is advisible to use 'from module import class' operator
# otherwise i could not find the module. But it is possible that
# in the future I can change this code to do it more wisely
raise TypeError( 'Can not find class ' + self.getType() + ' in the module created from the body of the module ' + self.getOrigin() )
else:
raise TypeError( 'The body of the Module ' + self.getType() + ' seems empty' )
return self.main_class_obj