def __init__( self, step_type = None, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setType( 'notgiven' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj ) # creating copy
self.module_instances = InstancesPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, StepDefinition ):
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
# copy instances and definitions
self.parameters = ParameterCollection( self, obj.parameters )
self.module_instances = InstancesPool( self, obj.module_instances )
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool( self. obj.module_definitions )
else:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if step_type :
self.setType( step_type )
def fromWorkflow(self, obj):
self.setName(obj.getName())
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(obj.parameters)
self.module_definitions = DefinitionsPool(self, obj.module_definitions)
self.step_instances = InstancesPool(self, obj.step_instances)
self.step_definitions = DefinitionsPool(self, obj.step_definitions)
def fromWorkflow( self, obj ): self.setName( obj.getName() ) self.setType( obj.getType() ) self.setDescrShort( obj.getDescrShort() ) self.setDescription( obj.getDescription() ) self.setOrigin( obj.getOrigin() ) self.setVersion( obj.getVersion() ) # copy instances and definitions self.parameters = ParameterCollection( obj.parameters ) self.module_definitions = DefinitionsPool( self, obj.module_definitions ) self.step_instances = InstancesPool( self, obj.step_instances ) self.step_definitions = DefinitionsPool( self, obj.step_definitions )
def __init__(self, obj=None, name=None):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__(self)
if (obj == None) or isinstance(obj, ParameterCollection):
self.setName('notgiven')
self.setType('')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, Workflow):
self.fromWorkflow(obj)
elif isinstance(obj, str):
self.parameters = ParameterCollection(None)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
# if obj is an XML string
if obj.startswith('<'):
fromXMLString(obj, self)
else:
fromXMLFile(obj, self)
elif obj != None:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if name:
self.setName(name)
self.workflow_commons = {}
self.workflowStatus = S_OK()
def __init__( self, obj = None, name = None ):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__( self )
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setName( 'notgiven' )
self.setType( '' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, Workflow ):
self.fromWorkflow( obj )
elif isinstance( obj, str ):
self.parameters = ParameterCollection( None )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
# if obj is an XML string
if obj.startswith( '<' ):
fromXMLString( obj, self )
else:
fromXMLFile( obj, self )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if name :
self.setName( name )
self.workflow_commons = {}
self.workflowStatus = S_OK()
def __init__(self, step_type=None, obj=None, parent=None):
AttributeCollection.__init__(self)
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if (obj == None) or isinstance(obj, ParameterCollection):
self.setType('notgiven')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj) # creating copy
self.module_instances = InstancesPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, StepDefinition):
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(self, obj.parameters)
self.module_instances = InstancesPool(self, obj.module_instances)
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool(
self.obj.module_definitions)
else:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if step_type:
self.setType(step_type)
class StepDefinition( AttributeCollection ):
def __init__( self, step_type = None, obj = None, parent = None ):
AttributeCollection.__init__( self )
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setType( 'notgiven' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj ) # creating copy
self.module_instances = InstancesPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, StepDefinition ):
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
# copy instances and definitions
self.parameters = ParameterCollection( self, obj.parameters )
self.module_instances = InstancesPool( self, obj.module_instances )
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool( self. obj.module_definitions )
else:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if step_type :
self.setType( step_type )
def __str__( self ):
ret = str( type( self ) ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
if self.module_definitions != None:
ret = ret + str( self.module_definitions )
else:
ret = ret + 'Module definitions shared in Workflow\n'
ret = ret + str( self.module_instances )
return ret
def toXML( self ):
ret = '<StepDefinition>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
if self.module_definitions != None:
ret = ret + self.module_definitions.toXML()
ret = ret + self.module_instances.toXML()
ret = ret + '</StepDefinition>\n'
return ret
def toXMLFile( self, outFile ):
if os.path.exists( outFile ):
os.remove( outFile )
xmlfile = open( outFile, 'w' )
xmlfile.write( self.toXML() )
xmlfile.close()
def addModule( self, module ):
# KGG We need to add code to update existing modules
if self.module_definitions == None:
self.parent.module_definitions.append( module )
else:
self.module_definitions.append( module )
return module
def createModuleInstance( self, module_type, name ):
''' Creates module instance of type 'type' with the name 'name'
'''
if self.module_definitions[module_type]:
mi = ModuleInstance( name, self.module_definitions[module_type] )
self.module_instances.append( mi )
return mi
else:
raise KeyError( 'Can not find ModuleDefinition ' + module_type + ' to create ModuleInstrance ' + name )
def removeModuleInstance( self, name ):
''' Remove module instance specified by its name
'''
self.module_instances.delete( name )
def compare( self, s ):
''' Custom Step comparison operation
'''
ret = AttributeCollection.compare( self, s ) and self.module_instances.compare( s )
if self.module_definitions.getOwner() == self:
ret = ret and self.module_definitions.compare( s )
return ret
def updateParents( self, parent ):
'''
'''
#AttributeCollection.updateParents( self, parent )
self.module_instances.updateParents( self )
if( self.module_definitions != None ):
self.module_definitions.updateParents( self )
def createCode( self ):
''' Create Step code
'''
str_ = 'class ' + self.getType() + ':\n'
str_ = str_ + indent( 1 ) + 'def execute(self):\n'
str_ = str_ + self.module_instances.createCode()
str_ = str_ + indent( 2 ) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode( 2, 'self' )
str_ += '\n'
return str_
class Workflow(AttributeCollection):
def __init__(self, obj=None, name=None):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__(self)
if (obj == None) or isinstance(obj, ParameterCollection):
self.setName('notgiven')
self.setType('')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, Workflow):
self.fromWorkflow(obj)
elif isinstance(obj, str):
self.parameters = ParameterCollection(None)
self.step_instances = InstancesPool(self)
self.step_definitions = DefinitionsPool(self)
self.module_definitions = DefinitionsPool(self)
# if obj is an XML string
if obj.startswith('<'):
fromXMLString(obj, self)
else:
fromXMLFile(obj, self)
elif obj != None:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if name:
self.setName(name)
self.workflow_commons = {}
self.workflowStatus = S_OK()
def fromWorkflow(self, obj):
self.setName(obj.getName())
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(obj.parameters)
self.module_definitions = DefinitionsPool(self, obj.module_definitions)
self.step_instances = InstancesPool(self, obj.step_instances)
self.step_definitions = DefinitionsPool(self, obj.step_definitions)
def __str__(self):
'''Creates a string representation of itself
'''
ret = str(self.getName()) + ':\n' + AttributeCollection.__str__(
self) + self.parameters.__str__()
ret = ret + str(self.step_definitions)
ret = ret + str(self.step_instances)
ret = ret + str(self.module_definitions)
return ret
def toXML(self):
'''Creates an XML representation of itself
'''
# THIS is very important that Definitions should be written before instances
ret = '<Workflow>\n'
ret = ret + AttributeCollection.toXML(self)
ret = ret + self.parameters.toXML()
ret = ret + self.module_definitions.toXML()
ret = ret + self.step_definitions.toXML()
ret = ret + self.step_instances.toXML()
ret = ret + '</Workflow>\n'
return ret
def toXMLFile(self, outFile):
if os.path.exists(outFile):
os.remove(outFile)
xmlfile = open(outFile, 'w')
xmlfile.write(self.toXML())
xmlfile.close()
def addTool(self, name, tool):
''' Add an object that will be available in all the modules to perform some operations.
For example, a state reporting facility.
'''
self.workflow_commons[name] = tool
def addStep(self, step):
# this is a VERY important piece of code
# we have to join all Modules definition from all added steps in the single dictionary
# and we have to share this dictionary between all included steps
# we also have to check versions of the modules and instances
for type_ in step.module_definitions.keys():
#if self.module_definitions.has_key(type):
# we have the same ModuleDefinition in 2 places
# we need to find way to synchronise it
#print "Workflow:addStep - we need to write ModuleDefinitions synchronisation code"
#else:
# new module - just append it
if not self.module_definitions.has_key(type_):
self.module_definitions.append(step.module_definitions[type_])
self.step_definitions.append(step)
del step.module_definitions # we need to clean up all unwanted definitions
step.module_definitions = None
return step
def addModule(self, module):
# KGG We need to add code to update existing modules
self.module_definitions.append(module)
return module
def createStepInstance(self, type_, name):
''' Creates step instance of type 'type' with the name 'name'
'''
if self.step_definitions.has_key(type_):
stepi = StepInstance(name, self.step_definitions[type_])
self.step_instances.append(stepi)
return stepi
else:
raise KeyError('Can not find StepDefinition ' + type_ +
' to create StepInstrance ' + name)
def removeStepInstance(self, name):
self.instances[name].setParents(None)
self.instances.delete(name)
def updateParents(self):
self.module_definitions.updateParents(self)
self.step_instances.updateParents(self)
self.step_definitions.updateParents(self)
def resolveGlobalVars(self):
''' This method will create global parameter list and then will resolve all instances of @{VARNAME}
Be aware that parameters of that type are GLOBAL!!! are string and can not be dynamically change
The scope: the resolution of that parameters apply from lower to upper object, for example if
parameter use in module, then it checks module, then step, then workflow
Comment: If varible linked it should not be used in a global list'''
# reenforced global parameters on the level of Workflow
if not self.parameters.find("PRODUCTION_ID"):
self.parameters.append(
Parameter(
"PRODUCTION_ID", "00000000", "string", "", "", True, False,
"Transformation ID taken from the ProductionManager"))
if not self.parameters.find("JOB_ID"):
self.parameters.append(
Parameter(
"JOB_ID", "00000000", "string", "", "", True, False,
"Job ID within Tranformationtaken from the ProductionManager"
))
self.parameters.resolveGlobalVars()
step_instance_number = 0
for inst in self.step_instances:
# for each step instance we can define STEP_NUMBER
step_instance_number = step_instance_number + 1
if not inst.parameters.find("STEP_NUMBER"):
inst.parameters.append(
Parameter(
"STEP_NUMBER", "%s" % step_instance_number, "string",
"", "", True, False,
"Number of the StepInstance within the Workflow"))
if not inst.parameters.find("STEP_ID"):
prod_ID = self.parameters.find("PRODUCTION_ID").getValue()
job_ID = self.parameters.find("JOB_ID").getValue()
inst.parameters.append(
Parameter(
"STEP_ID",
"%s_%s_%d" % (prod_ID, job_ID, step_instance_number),
"string", "", "", True, False, "Step instance ID"))
if not inst.parameters.find("STEP_INSTANCE_NAME"):
inst.parameters.append(
Parameter("STEP_INSTANCE_NAME", inst.getName(), "string",
"", "", True, False,
"Name of the StepInstance within the Workflow"))
if not inst.parameters.find("STEP_DEFINITION_NAME"):
inst.parameters.append(
Parameter("STEP_DEFINITION_NAME", inst.getType(), "string",
"", "", True, False,
"Type of the StepInstance within the Workflow"))
if not inst.parameters.find("JOB_ID"):
inst.parameters.append(
Parameter("JOB_ID", "", "string", "self", "JOB_ID", True,
False,
"Type of the StepInstance within the Workflow"))
if not inst.parameters.find("PRODUCTION_ID"):
inst.parameters.append(
Parameter("PRODUCTION_ID", "", "string", "self",
"PRODUCTION_ID", True, False,
"Type of the StepInstance within the Workflow"))
inst.resolveGlobalVars(self.step_definitions, self.parameters)
def createCode(self, combine_steps=False):
self.resolveGlobalVars()
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent(1) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent(2) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode(2, 'self')
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode(0, 'j')
str_ = str_ + 'j.execute()'
return str_
def showCode(self, combine_steps=False):
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent(1) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent(2) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode(2, 'self')
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode(0, 'j')
str_ = str_ + 'j.execute()'
return str_
def execute(self):
self.resolveGlobalVars()
# define workflow attributes
wf_exec_attr = {
} # dictianary with the WF attributes, used to resolve links to self.attrname
for wf_parameter in self.parameters:
# parameters shall see objects in the current scope order to resolve links
if wf_parameter.preExecute(): # for parm which not just outputs
# print 'Input', wf_parameter
if wf_parameter.isLinked():
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Job attribute ", wf_parameter.getName(
), "refers to the attribute of the same workflow", wf_parameter.getLinkedParameter(
)
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[
wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[
wf_parameter.getLinkedModule()][
wf_parameter.getLinkedParameter()]
else:
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getValue()
wf_exec_attr[
wf_parameter.getName()] = wf_parameter.getValue()
# Put all the workflow parameters into the workflow_commons dictionary
self.workflow_commons[
wf_parameter.getName()] = wf_parameter.getValue()
self.module_definitions.loadCode(
) # loading Module classes into current python scope
#wf_exec_steps will be dictionary of dictionaries [step instance name][parameter name]
# used as dictionary of step instances to carry parameters
wf_exec_steps = {}
# print 'Executing Workflow',self.getType()
error_message = ''
step_result = ''
for step_inst in self.step_instances:
step_inst_name = step_inst.getName()
wf_exec_steps[step_inst_name] = {}
# step_inst_type = step_inst.getType()
# print "WorkflowInstance creating Step instance ",step_inst_name," of type", step_inst_type
for parameter in step_inst.parameters:
if parameter.preExecute():
# print '>> Input', parameter
if parameter.isLinked():
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
wf_exec_steps[step_inst_name][parameter.getName(
)] = wf_exec_attr[parameter.getLinkedParameter()]
else:
# print wf_exec_steps[parameter.getLinkedModule()].keys()
wf_exec_steps[step_inst_name][parameter.getName(
)] = wf_exec_steps[parameter.getLinkedModule()][
parameter.getLinkedParameter()]
else:
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
wf_exec_steps[step_inst_name][
parameter.getName()] = parameter.getValue()
# In the step_commons all parameters are added, both Input and Output ones.
step_inst.step_commons[
parameter.getName()] = parameter.getValue()
resolveVariables(wf_exec_steps[step_inst_name])
# Set proper values for all Input Parameters
for key, value in wf_exec_steps[step_inst_name].items():
step_inst.step_commons[key] = value
step_inst.setParent(self)
step_inst.setWorkflowCommons(self.workflow_commons)
result = step_inst.execute(wf_exec_steps[step_inst_name],
self.step_definitions)
if not result['OK']:
if self.workflowStatus['OK']:
error_message = result['Message']
self.workflowStatus = S_ERROR(result['Message'])
if result.has_key('Value'):
step_result = result['Value']
# now we need to copy output values to the STEP!!! parameters
#print "WorkflowInstance output assignment"
for wf_parameter in self.parameters:
if wf_parameter.isOutput():
if wf_parameter.isLinked():
# print "WorkflowInstance self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Workflow OUTPUT attribute ", wf_parameter.getName(
), "refer on the attribute of the same workflow", wf_parameter.getLinkedParameter(
)
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[
wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_steps[
wf_parameter.getLinkedModule()][
wf_parameter.getLinkedParameter()]
else:
# it is also does not make sense - we can produce warning
print "Warning! Workflow OUTPUT attribute", wf_parameter.getName(
), "assigned constant", wf_parameter.getValue()
# print "WorkflowInstance self."+ wf_parameter.getName(),'=',wf_parameter.getValue()
wf_exec_attr[
wf_parameter.getName()] = wf_parameter.getValue()
setattr(self, wf_parameter.getName(),
wf_exec_attr[wf_parameter.getName()])
# Return the result of the first failed step or S_OK
if not self.workflowStatus['OK']:
return S_ERROR(error_message)
else:
return S_OK(step_result)
class Workflow( AttributeCollection ):
def __init__( self, obj = None, name = None ):
''' Be aware that 1-st param is an obj not a name!!!!
obj can me a string with XML representation or with filename
also obj can be a Workflow or ParameterCollections
'''
AttributeCollection.__init__( self )
if ( obj == None ) or isinstance( obj, ParameterCollection ):
self.setName( 'notgiven' )
self.setType( '' )
self.setDescrShort( '' )
self.setDescription( '' )
self.setOrigin( '' )
self.setVersion( 0.0 )
self.parameters = ParameterCollection( obj )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
elif isinstance( obj, Workflow ):
self.fromWorkflow( obj )
elif isinstance( obj, str ):
self.parameters = ParameterCollection( None )
self.step_instances = InstancesPool( self )
self.step_definitions = DefinitionsPool( self )
self.module_definitions = DefinitionsPool( self )
# if obj is an XML string
if obj.startswith( '<' ):
fromXMLString( obj, self )
else:
fromXMLFile( obj, self )
elif obj != None:
raise TypeError( 'Can not create object type ' + str( type( self ) ) + ' from the ' + str( type( obj ) ) )
if name :
self.setName( name )
self.workflow_commons = {}
self.workflowStatus = S_OK()
def fromWorkflow( self, obj ):
self.setName( obj.getName() )
self.setType( obj.getType() )
self.setDescrShort( obj.getDescrShort() )
self.setDescription( obj.getDescription() )
self.setOrigin( obj.getOrigin() )
self.setVersion( obj.getVersion() )
# copy instances and definitions
self.parameters = ParameterCollection( obj.parameters )
self.module_definitions = DefinitionsPool( self, obj.module_definitions )
self.step_instances = InstancesPool( self, obj.step_instances )
self.step_definitions = DefinitionsPool( self, obj.step_definitions )
def __str__( self ):
'''Creates a string representation of itself
'''
ret = str( self.getName() ) + ':\n' + AttributeCollection.__str__( self ) + self.parameters.__str__()
ret = ret + str( self.step_definitions )
ret = ret + str( self.step_instances )
ret = ret + str( self.module_definitions )
return ret
def toXML( self ):
'''Creates an XML representation of itself
'''
# THIS is very important that Definitions should be written before instances
ret = '<Workflow>\n'
ret = ret + AttributeCollection.toXML( self )
ret = ret + self.parameters.toXML()
ret = ret + self.module_definitions.toXML()
ret = ret + self.step_definitions.toXML()
ret = ret + self.step_instances.toXML()
ret = ret + '</Workflow>\n'
return ret
def toXMLFile( self, outFile ):
if os.path.exists( outFile ):
os.remove( outFile )
xmlfile = open( outFile, 'w' )
xmlfile.write( self.toXML() )
xmlfile.close()
def addTool( self, name, tool ):
''' Add an object that will be available in all the modules to perform some operations.
For example, a state reporting facility.
'''
self.workflow_commons[name] = tool
def addStep( self, step ):
# this is a VERY important piece of code
# we have to join all Modules definition from all added steps in the single dictionary
# and we have to share this dictionary between all included steps
# we also have to check versions of the modules and instances
for type_ in step.module_definitions.keys():
#if self.module_definitions.has_key(type):
# we have the same ModuleDefinition in 2 places
# we need to find way to synchronise it
#print "Workflow:addStep - we need to write ModuleDefinitions synchronisation code"
#else:
# new module - just append it
if not self.module_definitions.has_key( type_ ):
self.module_definitions.append( step.module_definitions[type_] )
self.step_definitions.append( step )
del step.module_definitions # we need to clean up all unwanted definitions
step.module_definitions = None
return step
def addModule( self, module ):
# KGG We need to add code to update existing modules
self.module_definitions.append( module )
return module
def createStepInstance( self, type_, name ):
''' Creates step instance of type 'type' with the name 'name'
'''
if self.step_definitions.has_key( type_ ):
stepi = StepInstance( name, self.step_definitions[type_] )
self.step_instances.append( stepi )
return stepi
else:
raise KeyError( 'Can not find StepDefinition ' + type_ + ' to create StepInstrance ' + name )
def removeStepInstance( self, name ):
self.instances[name].setParents( None )
self.instances.delete( name )
def updateParents( self ):
self.module_definitions.updateParents( self )
self.step_instances.updateParents( self )
self.step_definitions.updateParents( self )
def resolveGlobalVars( self ):
''' This method will create global parameter list and then will resolve all instances of @{VARNAME}
Be aware that parameters of that type are GLOBAL!!! are string and can not be dynamically change
The scope: the resolution of that parameters apply from lower to upper object, for example if
parameter use in module, then it checks module, then step, then workflow
Comment: If varible linked it should not be used in a global list'''
# reenforced global parameters on the level of Workflow
if not self.parameters.find( "PRODUCTION_ID" ):
self.parameters.append( Parameter( "PRODUCTION_ID", "00000000", "string", "", "", True, False, "Transformation ID taken from the ProductionManager" ) )
if not self.parameters.find( "JOB_ID" ):
self.parameters.append( Parameter( "JOB_ID", "00000000", "string", "", "", True, False, "Job ID within Tranformationtaken from the ProductionManager" ) )
self.parameters.resolveGlobalVars()
step_instance_number = 0
for inst in self.step_instances:
# for each step instance we can define STEP_NUMBER
step_instance_number = step_instance_number + 1
if not inst.parameters.find( "STEP_NUMBER" ):
inst.parameters.append( Parameter( "STEP_NUMBER", "%s" % step_instance_number, "string", "", "", True, False, "Number of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "STEP_ID" ):
prod_ID = self.parameters.find( "PRODUCTION_ID" ).getValue()
job_ID = self.parameters.find( "JOB_ID" ).getValue()
inst.parameters.append( Parameter( "STEP_ID", "%s_%s_%d" % ( prod_ID, job_ID, step_instance_number ), "string", "", "", True, False, "Step instance ID" ) )
if not inst.parameters.find( "STEP_INSTANCE_NAME" ):
inst.parameters.append( Parameter( "STEP_INSTANCE_NAME", inst.getName(), "string", "", "", True, False, "Name of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "STEP_DEFINITION_NAME" ):
inst.parameters.append( Parameter( "STEP_DEFINITION_NAME", inst.getType(), "string", "", "", True, False, "Type of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "JOB_ID" ):
inst.parameters.append( Parameter( "JOB_ID", "", "string", "self", "JOB_ID", True, False, "Type of the StepInstance within the Workflow" ) )
if not inst.parameters.find( "PRODUCTION_ID" ):
inst.parameters.append( Parameter( "PRODUCTION_ID", "", "string", "self", "PRODUCTION_ID", True, False, "Type of the StepInstance within the Workflow" ) )
inst.resolveGlobalVars( self.step_definitions, self.parameters )
def createCode( self, combine_steps = False ):
self.resolveGlobalVars()
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent( 1 ) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent( 2 ) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode( 2, 'self' )
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode( 0, 'j' )
str_ = str_ + 'j.execute()'
return str_
def showCode( self, combine_steps = False ):
str_ = ''
str_ = str_ + self.module_definitions.createCode()
str_ = str_ + self.step_definitions.createCode()
str_ = str_ + "\nclass job:\n"
str_ = str_ + indent( 1 ) + 'def execute(self):\n'
#str_=str_+indent(2)+'# flush self.step_instances\n'
str_ = str_ + self.step_instances.createCode()
# it seems we do not need it on this level
str_ = str_ + indent( 2 ) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode( 2, 'self' )
str_ = str_ + '\nj=job()\n'
str_ = str_ + self.parameters.createParametersCode( 0, 'j' )
str_ = str_ + 'j.execute()'
return str_
def execute( self ):
self.resolveGlobalVars()
# define workflow attributes
wf_exec_attr = {} # dictianary with the WF attributes, used to resolve links to self.attrname
for wf_parameter in self.parameters:
# parameters shall see objects in the current scope order to resolve links
if wf_parameter.preExecute(): # for parm which not just outputs
# print 'Input', wf_parameter
if wf_parameter.isLinked():
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Job attribute ", wf_parameter.getName(), "refers to the attribute of the same workflow", wf_parameter.getLinkedParameter()
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedModule()][wf_parameter.getLinkedParameter()]
else:
# print "Workflow self." + wf_parameter.getName(), '=', wf_parameter.getValue()
wf_exec_attr[wf_parameter.getName()] = wf_parameter.getValue()
# Put all the workflow parameters into the workflow_commons dictionary
self.workflow_commons[wf_parameter.getName()] = wf_parameter.getValue()
self.module_definitions.loadCode() # loading Module classes into current python scope
#wf_exec_steps will be dictionary of dictionaries [step instance name][parameter name]
# used as dictionary of step instances to carry parameters
wf_exec_steps = {}
# print 'Executing Workflow',self.getType()
error_message = ''
step_result = ''
for step_inst in self.step_instances:
step_inst_name = step_inst.getName()
wf_exec_steps[step_inst_name] = {}
# step_inst_type = step_inst.getType()
# print "WorkflowInstance creating Step instance ",step_inst_name," of type", step_inst_type
for parameter in step_inst.parameters:
if parameter.preExecute():
# print '>> Input', parameter
if parameter.isLinked():
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getLinkedModule() + '.' + parameter.getLinkedParameter()
if parameter.getLinkedModule() == 'self':
# tale value form the step_dict
wf_exec_steps[step_inst_name][parameter.getName()] = wf_exec_attr[parameter.getLinkedParameter()]
else:
# print wf_exec_steps[parameter.getLinkedModule()].keys()
wf_exec_steps[step_inst_name][parameter.getName()] = wf_exec_steps[parameter.getLinkedModule()][parameter.getLinkedParameter()]
else:
# print ">> StepInstance", step_inst_name + '.' + parameter.getName(), '=', parameter.getValue()
wf_exec_steps[step_inst_name][parameter.getName()] = parameter.getValue()
# In the step_commons all parameters are added, both Input and Output ones.
step_inst.step_commons[parameter.getName()] = parameter.getValue()
resolveVariables( wf_exec_steps[step_inst_name] )
# Set proper values for all Input Parameters
for key, value in wf_exec_steps[step_inst_name].items():
step_inst.step_commons[key] = value
step_inst.setParent( self )
step_inst.setWorkflowCommons( self.workflow_commons )
result = step_inst.execute( wf_exec_steps[step_inst_name], self.step_definitions )
if not result['OK']:
if self.workflowStatus['OK']:
error_message = result['Message']
self.workflowStatus = S_ERROR( result['Message'] )
if result.has_key( 'Value' ):
step_result = result['Value']
# now we need to copy output values to the STEP!!! parameters
#print "WorkflowInstance output assignment"
for wf_parameter in self.parameters:
if wf_parameter.isOutput():
if wf_parameter.isLinked():
# print "WorkflowInstance self." + wf_parameter.getName(), '=', wf_parameter.getLinkedModule() + '.' + wf_parameter.getLinkedParameter()
if wf_parameter.getLinkedModule() == 'self':
# this is not suppose to happen
print "Warning! Workflow OUTPUT attribute ", wf_parameter.getName(), "refer on the attribute of the same workflow", wf_parameter.getLinkedParameter()
wf_exec_attr[wf_parameter.getName()] = wf_exec_attr[wf_parameter.getLinkedParameter()]
else:
wf_exec_attr[wf_parameter.getName()] = wf_exec_steps[wf_parameter.getLinkedModule()][wf_parameter.getLinkedParameter()]
else:
# it is also does not make sense - we can produce warning
print "Warning! Workflow OUTPUT attribute", wf_parameter.getName(), "assigned constant", wf_parameter.getValue()
# print "WorkflowInstance self."+ wf_parameter.getName(),'=',wf_parameter.getValue()
wf_exec_attr[wf_parameter.getName()] = wf_parameter.getValue()
setattr( self, wf_parameter.getName(), wf_exec_attr[wf_parameter.getName()] )
# Return the result of the first failed step or S_OK
if not self.workflowStatus['OK']:
return S_ERROR( error_message )
else:
return S_OK( step_result )
class StepDefinition(AttributeCollection):
def __init__(self, step_type=None, obj=None, parent=None):
AttributeCollection.__init__(self)
self.module_instances = None
# this object can be shared with the workflow
# to if its =None and workflow!=None we have to
# pass everything above
self.module_definitions = None
self.parent = parent
# sort out Parameters and class attributes
if (obj == None) or isinstance(obj, ParameterCollection):
self.setType('notgiven')
self.setDescrShort('')
self.setDescription('')
self.setOrigin('')
self.setVersion(0.0)
self.parameters = ParameterCollection(obj) # creating copy
self.module_instances = InstancesPool(self)
self.module_definitions = DefinitionsPool(self)
elif isinstance(obj, StepDefinition):
self.setType(obj.getType())
self.setDescrShort(obj.getDescrShort())
self.setDescription(obj.getDescription())
self.setOrigin(obj.getOrigin())
self.setVersion(obj.getVersion())
# copy instances and definitions
self.parameters = ParameterCollection(self, obj.parameters)
self.module_instances = InstancesPool(self, obj.module_instances)
if obj.module_definitions != None:
self.module_definitions = DefinitionsPool(
self.obj.module_definitions)
else:
raise TypeError('Can not create object type ' + str(type(self)) +
' from the ' + str(type(obj)))
if step_type:
self.setType(step_type)
def __str__(self):
ret = str(type(self)) + ':\n' + AttributeCollection.__str__(
self) + self.parameters.__str__()
if self.module_definitions != None:
ret = ret + str(self.module_definitions)
else:
ret = ret + 'Module definitions shared in Workflow\n'
ret = ret + str(self.module_instances)
return ret
def toXML(self):
ret = '<StepDefinition>\n'
ret = ret + AttributeCollection.toXML(self)
ret = ret + self.parameters.toXML()
if self.module_definitions != None:
ret = ret + self.module_definitions.toXML()
ret = ret + self.module_instances.toXML()
ret = ret + '</StepDefinition>\n'
return ret
def toXMLFile(self, outFile):
if os.path.exists(outFile):
os.remove(outFile)
xmlfile = open(outFile, 'w')
xmlfile.write(self.toXML())
xmlfile.close()
def addModule(self, module):
# KGG We need to add code to update existing modules
if self.module_definitions == None:
self.parent.module_definitions.append(module)
else:
self.module_definitions.append(module)
return module
def createModuleInstance(self, module_type, name):
''' Creates module instance of type 'type' with the name 'name'
'''
if self.module_definitions[module_type]:
mi = ModuleInstance(name, self.module_definitions[module_type])
self.module_instances.append(mi)
return mi
else:
raise KeyError('Can not find ModuleDefinition ' + module_type +
' to create ModuleInstrance ' + name)
def removeModuleInstance(self, name):
''' Remove module instance specified by its name
'''
self.module_instances.delete(name)
def compare(self, s):
''' Custom Step comparison operation
'''
ret = AttributeCollection.compare(
self, s) and self.module_instances.compare(s)
if self.module_definitions.getOwner() == self:
ret = ret and self.module_definitions.compare(s)
return ret
def updateParents(self, parent):
'''
'''
#AttributeCollection.updateParents( self, parent )
self.module_instances.updateParents(self)
if (self.module_definitions != None):
self.module_definitions.updateParents(self)
def createCode(self):
''' Create Step code
'''
str_ = 'class ' + self.getType() + ':\n'
str_ = str_ + indent(1) + 'def execute(self):\n'
str_ = str_ + self.module_instances.createCode()
str_ = str_ + indent(2) + '# output assignment\n'
for v in self.parameters:
if v.isOutput():
str_ = str_ + v.createParameterCode(2, 'self')
str_ += '\n'
return str_