def __generateSimulationCondition(self, jobNode):
'''SimulationCondition looks like this:
<SimulationCondition>
<Parameter Name="" Value=""/>
</SimulationCondition>
'''
if self.applicationName == "Gauss":
jobNode, sim = addChildNode(jobNode, "SimulationCondition", 1, ())
sim = addChildNode(sim, "Parameter", 0,
("SimDescription", self.simDescription))
return jobNode
def __generateInputFiles(self, jobNode, bkLFNs):
''' InputData looks like this
<InputFile Name=""/>
'''
if self.stepInputData:
intermediateInputs = False
for inputname in self.stepInputData:
for bkLFN in bkLFNs:
if os.path.basename(bkLFN) == os.path.basename(inputname):
jobNode = addChildNode(jobNode, "InputFile", 0,
(bkLFN, ))
intermediateInputs = True
if not intermediateInputs:
jobNode = addChildNode(jobNode, "InputFile", 0,
(inputname, ))
return jobNode
def __generateJobNode(self, doc):
''' Node looks like
<Job ConfigName="" ConfigVersion="" Date="" Time="">
'''
# Get the Config name from the environment if any
if 'configName' in self.workflow_commons:
configName = self.workflow_commons['configName']
configVersion = self.workflow_commons['configVersion']
else:
configName = self.applicationName
configVersion = self.applicationVersion
jobAttributes = (configName, configVersion, self.ldate, self.ltime)
return addChildNode(doc, "Job", 1, jobAttributes)
def __generateOutputFiles(self, jobNode, bkLFNs, logFilePath):
'''OutputFile looks like this:
<OutputFile Name="" TypeName="" TypeVersion="">
<Parameter Name="" Value=""/>
...
<Replica Location="" Name=""/>
....
</OutputFile>
'''
if self.eventType is not None:
eventtype = self.eventType
else:
self.log.warn('BookkeepingReport: no eventType specified')
eventtype = 'Unknown'
self.log.info('Event type = %s' % (str(self.eventType)))
outputs = []
count = 0
bkTypeDict = {}
while count < len(self.stepOutputs):
if 'outputDataName' in self.stepOutputs[count]:
outputs.append(((self.stepOutputs[count]['outputDataName']),
(self.stepOutputs[count]['outputDataType'])))
if 'outputBKType' in self.stepOutputs[count]:
bkTypeDict[self.stepOutputs[count]
['outputDataName']] = self.stepOutputs[count][
'outputBKType']
count = count + 1
outputs.append(((self.applicationLog), ('LOG')))
self.log.info(outputs)
if isinstance(logFilePath, list):
logFilePath = logFilePath[0]
for output, outputtype in list(outputs):
self.log.info('Looking at output %s %s' % (output, outputtype))
typeName = outputtype.upper()
typeVersion = '1'
fileStats = '0'
if output in bkTypeDict:
typeVersion = getOutputType(output, self.stepInputData)[output]
self.log.info('Setting POOL XML catalog type',
'for %s to %s' % (output, typeVersion))
typeName = bkTypeDict[output].upper()
self.log.info(
'Setting explicit BK type version',
'for %s to %s and file type to %s' %
(output, typeVersion, typeName))
fileStats, output = self._getFileStatsFromXMLSummary(
output, outputtype)
if not os.path.exists(output):
self.log.error("Output file does not exist",
"Output file name: %s" % output)
continue
# Output file size
if 'size' not in self.step_commons or output not in self.step_commons[
'size']:
try:
outputsize = str(os.path.getsize(output))
except OSError:
outputsize = '0'
else:
outputsize = self.step_commons['size'][output]
if 'md5' not in self.step_commons or output not in self.step_commons[
'md5']:
comm = 'md5sum ' + str(output)
resultTuple = systemCall(0, shlex.split(comm))
status = resultTuple['Value'][0]
out = resultTuple['Value'][1]
else:
status = 0
out = self.step_commons['md5'][output]
if status:
self.log.info("Failed to get md5sum of %s" % str(output))
self.log.info(str(out))
md5sum = '000000000000000000000000000000000000'
else:
md5sum = out.split()[0]
if 'guid' not in self.step_commons or output not in self.step_commons[
'guid']:
guidResult = getGUID(output)
guid = ''
if not guidResult['OK']:
self.log.error(
"Could not find GUID", "for %s with message %s" %
(output, guidResult['Message']))
elif guidResult['generated']:
self.log.warn(
'PoolXMLFile generated GUID(s) for the following files ',
', '.join(guidResult['generated']))
guid = guidResult['Value'][output]
else:
guid = guidResult['Value'][output]
self.log.info('Setting POOL XML catalog GUID',
'for %s to %s' % (output, guid))
else:
guid = self.step_commons['guid'][output]
if not guid:
self.log.error('No GUID found', 'for %s' % output)
raise NameError('No GUID found')
# find the constructed lfn
lfn = ''
if not re.search('.log$', output):
for outputLFN in bkLFNs:
if os.path.basename(outputLFN) == output:
lfn = outputLFN
if not lfn:
self.log.error('Could not find LFN', 'for %s' % output)
raise NameError('Could not find LFN of output file')
else:
lfn = '%s/%s' % (logFilePath, self.applicationLog)
oldTypeName = None
if 'HIST' in typeName.upper():
typeVersion = '0'
# PROTECTION for old production XMLs
if typeName.upper() == 'HIST':
typeName = '%sHIST' % (self.applicationName.upper())
# Add Output to the XML file
oFileAttributes = (lfn, typeName, typeVersion)
jobNode, oFile = addChildNode(jobNode, "OutputFile", 1,
oFileAttributes)
# HIST is in the dataTypes e.g. we may have new names in the future ;)
if oldTypeName:
typeName = oldTypeName
if outputtype != 'LOG':
oFile = addChildNode(oFile, "Parameter", 0,
("EventTypeId", eventtype))
if fileStats != 'Unknown':
oFile = addChildNode(oFile, "Parameter", 0,
("EventStat", fileStats))
oFile = addChildNode(oFile, "Parameter", 0,
("FileSize", outputsize))
############################################################
# Log file replica information
# if typeName == "LOG":
if self.applicationLog:
logfile = self.applicationLog
if logfile == output:
logurl = 'http://lhcb-logs.cern.ch/storage'
url = logurl + logFilePath + '/' + self.applicationLog
oFile = addChildNode(oFile, "Replica", 0, (url, ))
oFile = addChildNode(oFile, "Parameter", 0, ("MD5Sum", md5sum))
oFile = addChildNode(oFile, "Parameter", 0, ("Guid", guid))
return jobNode
def __generateTypedParams(self, jobNode):
""" TypedParameter looks like
<TypedParameter Name="" Type="" Value="">
List of possible TypedParameter names
- CPUTIME
- ExecTime
- WNMODEL
- WNMEMORY
- WNCPUPOWER
- WNCACHE
- WNCPUHS06
- WNMJFHS06
- Production
- DiracJobId
- Name
- JobStart
- JobEnd
- Location
- JobType
- WorkerNode
- GeometryVersion
- ProgramName
- ProgramVersion
- DiracVersion
- FirstEventNumber
- StatisticsRequested
- NumberOfEvents
- StepID
"""
typedParams = []
# Timing
exectime, cputime = getStepCPUTimes(self.step_commons)
typedParams.append(("CPUTIME", cputime))
typedParams.append(("ExecTime", exectime))
nodeInfo = self.__getNodeInformation()
if nodeInfo['OK']:
typedParams.append(("WNMODEL", nodeInfo["ModelName"]))
typedParams.append(("WNCPUPOWER", nodeInfo["CPU(MHz)"]))
typedParams.append(("WNCACHE", nodeInfo["CacheSize(kB)"]))
# THIS OR THE NEXT
# typedParams.append( ( "WorkerNode", nodeInfo['HostName'] ) )
host = os.environ.get("HOSTNAME", os.environ.get("HOST"))
if host is not None:
typedParams.append(("WorkerNode", host))
try:
memory = self.xf_o.memory
except AttributeError:
memory = nodeInfo["Memory(kB)"]
typedParams.append(("WNMEMORY", memory))
diracPower = gConfig.getValue("/LocalSite/CPUNormalizationFactor", "0")
typedParams.append(("WNCPUHS06", diracPower))
mjfPower = gConfig.getValue("/LocalSite/CPUScalingFactor", "0")
# Trick to know that the value is obtained from MJF: # from diracPower
if mjfPower != diracPower:
typedParams.append(("WNMJFHS06", mjfPower))
typedParams.append(("Production", self.production_id))
typedParams.append(("DiracJobId", str(self.jobID)))
typedParams.append(("Name", self.step_id))
typedParams.append(
("JobStart", '%s %s' % (self.ldatestart, self.ltimestart)))
typedParams.append(("JobEnd", '%s %s' % (self.ldate, self.ltime)))
typedParams.append(("Location", self.siteName))
typedParams.append(("JobType", self.jobType))
if 'XMLDDDB_VERSION' in os.environ:
typedParams.append(
("GeometryVersion", os.environ["XMLDDDB_VERSION"]))
typedParams.append(("ProgramName", self.applicationName))
typedParams.append(("ProgramVersion", self.applicationVersion))
# DIRAC version
tempVar = "v%dr%dp%d" % (LHCbDIRAC.majorVersion,
LHCbDIRAC.minorVersion, LHCbDIRAC.patchLevel)
typedParams.append(("DiracVersion", tempVar))
typedParams.append(("FirstEventNumber", 1))
typedParams.append(("StatisticsRequested", self.numberOfEvents))
typedParams.append(("StepID", self.BKstepID))
try:
noOfEvents = self.xf_o.inputEventsTotal if self.xf_o.inputEventsTotal else self.xf_o.outputEventsTotal
except AttributeError:
# This happens iff the XML summary can't be created (e.g. for merging MDF files)
res = self.bkClient.getFileMetadata(self.stepInputData)
if not res['OK']:
raise AttributeError("Can't get the BKK file metadata")
noOfEvents = sum(
fileMeta['EventStat']
for fileMeta in res['Value']['Successful'].itervalues())
typedParams.append(("NumberOfEvents", noOfEvents))
# Add TypedParameters to the XML file
for typedParam in typedParams:
jobNode = addChildNode(jobNode, "TypedParameter", 0, typedParam)
return jobNode