def initialize(self):
"""Sets default parameters and creates CE instance"""
# Disable monitoring
self.am_disableMonitoring()
localCE = gConfig.getValue("/LocalSite/LocalCE", self.ceName)
if localCE != self.ceName:
self.log.info("Defining Inner CE from local configuration",
"= %s" % localCE)
# Create backend Computing Element
result = self._initializeComputingElement(localCE)
if not result["OK"]:
return result
result = self._getCEDict(self.computingElement)
if not result["OK"]:
return result
ceDict = result["Value"][0]
self.initTimeLeft = ceDict.get("CPUTime", self.initTimeLeft)
self.initTimeLeft = gConfig.getValue(
"/Resources/Computing/CEDefaults/MaxCPUTime", self.initTimeLeft)
self.timeLeft = self.initTimeLeft
self.initTimes = os.times()
# Localsite options
self.siteName = siteName()
self.pilotReference = gConfig.getValue("/LocalSite/PilotReference",
self.pilotReference)
self.defaultProxyLength = gConfig.getValue(
"/Registry/DefaultProxyLifeTime", self.defaultProxyLength)
# Agent options
# This is the factor to convert raw CPU to Normalized units (based on the CPU Model)
self.cpuFactor = gConfig.getValue("/LocalSite/CPUNormalizationFactor",
self.cpuFactor)
self.jobSubmissionDelay = self.am_getOption("SubmissionDelay",
self.jobSubmissionDelay)
self.fillingMode = self.am_getOption("FillingModeFlag",
self.fillingMode)
self.minimumTimeLeft = self.am_getOption("MinimumTimeLeft",
self.minimumTimeLeft)
self.stopOnApplicationFailure = self.am_getOption(
"StopOnApplicationFailure", self.stopOnApplicationFailure)
self.stopAfterFailedMatches = self.am_getOption(
"StopAfterFailedMatches", self.stopAfterFailedMatches)
self.extraOptions = gConfig.getValue(
"/AgentJobRequirements/ExtraOptions", self.extraOptions)
# Utilities
self.timeLeftUtil = TimeLeft()
return S_OK()
def execute(self, version=None,
production_id=None, prod_job_id=None, wms_job_id=None,
workflowStatus=None, stepStatus=None,
wf_commons=None, step_commons=None,
step_number=None, step_id=None):
""" Function called by all super classes
"""
if version:
self.log.info('===== Executing ' + version + ' ===== ')
self.log.verbose("Executing directory for job is %s" % os.getcwd())
if production_id:
self.production_id = production_id
else:
self.production_id = self.workflow_commons['PRODUCTION_ID'] # This is a string, like '00051753'
if prod_job_id:
self.prod_job_id = prod_job_id
else:
self.prod_job_id = self.workflow_commons['JOB_ID']
if 'JOBID' in os.environ:
self.jobID = os.environ['JOBID']
if wms_job_id:
self.jobID = wms_job_id
if workflowStatus:
self.workflowStatus = workflowStatus
if stepStatus:
self.stepStatus = stepStatus
if wf_commons:
self.workflow_commons = wf_commons
if step_commons:
self.step_commons = step_commons
if step_number:
self.step_number = step_number
else:
self.step_number = self.STEP_NUMBER # pylint: disable=no-member
if step_id:
self.step_id = step_id
else:
self.step_id = '%s_%s_%s' % (self.production_id, self.prod_job_id, self.step_number)
self.siteName = siteName()
def __newAccountingDict(self, storageElement):
""" Generate a new accounting Dict """
accountingDict = {}
accountingDict['OperationType'] = 'Stage'
accountingDict['User'] = self.proxyInfoDict['username']
accountingDict['Protocol'] = 'Stager'
accountingDict['RegistrationTime'] = 0.0
accountingDict['RegistrationOK'] = 0
accountingDict['RegistrationTotal'] = 0
accountingDict['FinalStatus'] = 'Successful'
accountingDict['Source'] = storageElement
accountingDict['Destination'] = storageElement
accountingDict['ExecutionSite'] = siteName()
accountingDict['TransferTotal'] = 0
accountingDict['TransferOK'] = 0
accountingDict['TransferSize'] = 0
accountingDict['TransferTime'] = self.am_getPollingTime()
return accountingDict
def __newAccountingDict( self, storageElement ):
""" Generate a new accounting Dict """
accountingDict = {}
accountingDict['OperationType'] = 'Stage'
accountingDict['User'] = self.proxyInfoDict['username']
accountingDict['Protocol'] = 'Stager'
accountingDict['RegistrationTime'] = 0.0
accountingDict['RegistrationOK'] = 0
accountingDict['RegistrationTotal'] = 0
accountingDict['FinalStatus'] = 'Successful'
accountingDict['Source'] = storageElement
accountingDict['Destination'] = storageElement
accountingDict['ExecutionSite'] = siteName()
accountingDict['TransferTotal'] = 0
accountingDict['TransferOK'] = 0
accountingDict['TransferSize'] = 0
accountingDict['TransferTime'] = self.am_getPollingTime()
return accountingDict
def __newAccountingDict(self, storageElement):
"""Generate a new accounting Dict"""
accountingDict = {}
accountingDict["OperationType"] = "Stage"
accountingDict["User"] = self.proxyInfoDict["username"]
accountingDict["Protocol"] = "Stager"
accountingDict["RegistrationTime"] = 0.0
accountingDict["RegistrationOK"] = 0
accountingDict["RegistrationTotal"] = 0
accountingDict["FinalStatus"] = "Successful"
accountingDict["Source"] = storageElement
accountingDict["Destination"] = storageElement
accountingDict["ExecutionSite"] = siteName()
accountingDict["TransferTotal"] = 0
accountingDict["TransferOK"] = 0
accountingDict["TransferSize"] = 0
accountingDict["TransferTime"] = self.am_getPollingTime()
return accountingDict
def __newAccountingDict(self, storageElement):
""" Generate a new accounting Dict """
accountingDict = {}
accountingDict["OperationType"] = "Stage"
accountingDict["User"] = self.proxyInfoDict["username"]
accountingDict["Protocol"] = "Stager"
accountingDict["RegistrationTime"] = 0.0
accountingDict["RegistrationOK"] = 0
accountingDict["RegistrationTotal"] = 0
accountingDict["FinalStatus"] = "Successful"
accountingDict["Source"] = storageElement
accountingDict["Destination"] = storageElement
accountingDict["ExecutionSite"] = siteName()
accountingDict["TransferTotal"] = 0
accountingDict["TransferOK"] = 0
accountingDict["TransferSize"] = 0
accountingDict["TransferTime"] = self.am_getPollingTime()
return accountingDict
def __initialiseAccountingObject( self, operation, se, startTime, endTime, size ):
accountingDict = {}
accountingDict['OperationType'] = operation
accountingDict['User'] = self.userName
accountingDict['Protocol'] = 'SRM'
accountingDict['RegistrationTime'] = 0.0
accountingDict['RegistrationOK'] = 0
accountingDict['RegistrationTotal'] = 0
accountingDict['TransferTotal'] = 1
accountingDict['TransferOK'] = 1
accountingDict['TransferSize'] = size
timeDiff = endTime - startTime
transferTime = ( timeDiff.days * 86400 ) + ( timeDiff.seconds ) + ( timeDiff.microseconds / 1000000.0 )
accountingDict['TransferTime'] = transferTime
accountingDict['FinalStatus'] = 'Successful'
accountingDict['Source'] = siteName()
accountingDict['Destination'] = se
oDataOperation = DataOperation()
oDataOperation.setEndTime( endTime )
oDataOperation.setStartTime( startTime )
oDataOperation.setValuesFromDict( accountingDict )
return oDataOperation
def __initialiseAccountingObject(self, operation, se, startTime, endTime,
size):
accountingDict = {}
accountingDict['OperationType'] = operation
accountingDict['User'] = self.userName
accountingDict['Protocol'] = 'SRM'
accountingDict['RegistrationTime'] = 0.0
accountingDict['RegistrationOK'] = 0
accountingDict['RegistrationTotal'] = 0
accountingDict['TransferTotal'] = 1
accountingDict['TransferOK'] = 1
accountingDict['TransferSize'] = size
timeDiff = endTime - startTime
transferTime = (timeDiff.days * 86400) + (timeDiff.seconds) + (
timeDiff.microseconds / 1000000.0)
accountingDict['TransferTime'] = transferTime
accountingDict['FinalStatus'] = 'Successful'
accountingDict['Source'] = siteName()
accountingDict['Destination'] = se
oDataOperation = DataOperation()
oDataOperation.setEndTime(endTime)
oDataOperation.setStartTime(startTime)
oDataOperation.setValuesFromDict(accountingDict)
return oDataOperation
def addAccountingOperation(self, lfns, startDate, elapsedTime, storageParameters, callRes):
"""
Generates a DataOperation accounting if needs to be, and adds it to the DataStore client cache
:param lfns : list of lfns on which we attempted the operation
:param startDate : datetime, start of the operation
:param elapsedTime : time (seconds) the operation took
:param storageParameters : the parameters of the plugins used to perform the operation
:param callRes : the return of the method call, S_OK or S_ERROR
The operation is generated with the OperationType "se.methodName"
The TransferSize and TransferTotal for directory methods actually take into
account the files inside the directory, and not the amount of directory given
as parameter
"""
if self.methodName not in (self.readMethods + self.writeMethods + self.removeMethods):
return
baseAccountingDict = {}
baseAccountingDict['OperationType'] = 'se.%s' % self.methodName
baseAccountingDict['User'] = getProxyInfo().get('Value', {}).get('username', 'unknown')
baseAccountingDict['RegistrationTime'] = 0.0
baseAccountingDict['RegistrationOK'] = 0
baseAccountingDict['RegistrationTotal'] = 0
# if it is a get method, then source and destination of the transfer should be inverted
if self.methodName == 'getFile':
baseAccountingDict['Destination'] = siteName()
baseAccountingDict['Source'] = self.name
else:
baseAccountingDict['Destination'] = self.name
baseAccountingDict['Source'] = siteName()
baseAccountingDict['TransferTotal'] = 0
baseAccountingDict['TransferOK'] = 0
baseAccountingDict['TransferSize'] = 0
baseAccountingDict['TransferTime'] = 0.0
baseAccountingDict['FinalStatus'] = 'Successful'
oDataOperation = DataOperation()
oDataOperation.setValuesFromDict(baseAccountingDict)
oDataOperation.setStartTime(startDate)
oDataOperation.setEndTime(startDate + datetime.timedelta(seconds=elapsedTime))
oDataOperation.setValueByKey('TransferTime', elapsedTime)
oDataOperation.setValueByKey('Protocol', storageParameters.get('Protocol', 'unknown'))
if not callRes['OK']:
# Everything failed
oDataOperation.setValueByKey('TransferTotal', len(lfns))
oDataOperation.setValueByKey('FinalStatus', 'Failed')
else:
succ = callRes.get('Value', {}).get('Successful', {})
failed = callRes.get('Value', {}).get('Failed', {})
totalSize = 0
# We don't take len(lfns) in order to make two
# separate entries in case of few failures
totalSucc = len(succ)
if self.methodName in ('putFile', 'getFile'):
# putFile and getFile return for each entry
# in the successful dir the size of the corresponding file
totalSize = sum(succ.values())
elif self.methodName in ('putDirectory', 'getDirectory'):
# putDirectory and getDirectory return for each dir name
# a dictionnary with the keys 'Files' and 'Size'
totalSize = sum(val.get('Size', 0) for val in succ.values() if isinstance(val, dict))
totalSucc = sum(val.get('Files', 0) for val in succ.values() if isinstance(val, dict))
oDataOperation.setValueByKey('TransferOK', len(succ))
oDataOperation.setValueByKey('TransferSize', totalSize)
oDataOperation.setValueByKey('TransferTotal', totalSucc)
oDataOperation.setValueByKey('TransferOK', totalSucc)
if callRes['Value']['Failed']:
oDataOperationFailed = copy.deepcopy(oDataOperation)
oDataOperationFailed.setValueByKey('TransferTotal', len(failed))
oDataOperationFailed.setValueByKey('TransferOK', 0)
oDataOperationFailed.setValueByKey('TransferSize', 0)
oDataOperationFailed.setValueByKey('FinalStatus', 'Failed')
accRes = gDataStoreClient.addRegister(oDataOperationFailed)
if not accRes['OK']:
self.log.error("Could not send failed accounting report", accRes['Message'])
accRes = gDataStoreClient.addRegister(oDataOperation)
if not accRes['OK']:
self.log.error("Could not send accounting report", accRes['Message'])
def addAccountingOperation(self, lfns, startDate, elapsedTime,
storageParameters, callRes):
"""
Generates a DataOperation accounting if needs to be, and adds it to the DataStore client cache
:param lfns : list of lfns on which we attempted the operation
:param startDate : datetime, start of the operation
:param elapsedTime : time (seconds) the operation took
:param storageParameters : the parameters of the plugins used to perform the operation
:param callRes : the return of the method call, S_OK or S_ERROR
The operation is generated with the OperationType "se.methodName"
The TransferSize and TransferTotal for directory methods actually take into
account the files inside the directory, and not the amount of directory given
as parameter
"""
if self.methodName not in (self.readMethods + self.writeMethods +
self.removeMethods):
return
baseAccountingDict = {}
baseAccountingDict['OperationType'] = 'se.%s' % self.methodName
baseAccountingDict['User'] = getProxyInfo().get('Value', {}).get(
'username', 'unknown')
baseAccountingDict['RegistrationTime'] = 0.0
baseAccountingDict['RegistrationOK'] = 0
baseAccountingDict['RegistrationTotal'] = 0
# if it is a get method, then source and destination of the transfer should be inverted
if self.methodName == 'getFile':
baseAccountingDict['Destination'] = siteName()
baseAccountingDict['Source'] = self.name
else:
baseAccountingDict['Destination'] = self.name
baseAccountingDict['Source'] = siteName()
baseAccountingDict['TransferTotal'] = 0
baseAccountingDict['TransferOK'] = 0
baseAccountingDict['TransferSize'] = 0
baseAccountingDict['TransferTime'] = 0.0
baseAccountingDict['FinalStatus'] = 'Successful'
oDataOperation = DataOperation()
oDataOperation.setValuesFromDict(baseAccountingDict)
oDataOperation.setStartTime(startDate)
oDataOperation.setEndTime(startDate +
datetime.timedelta(seconds=elapsedTime))
oDataOperation.setValueByKey('TransferTime', elapsedTime)
oDataOperation.setValueByKey(
'Protocol', storageParameters.get('Protocol', 'unknown'))
if not callRes['OK']:
# Everything failed
oDataOperation.setValueByKey('TransferTotal', len(lfns))
oDataOperation.setValueByKey('FinalStatus', 'Failed')
else:
succ = callRes.get('Value', {}).get('Successful', {})
failed = callRes.get('Value', {}).get('Failed', {})
totalSize = 0
# We don't take len(lfns) in order to make two
# separate entries in case of few failures
totalSucc = len(succ)
if self.methodName in ('putFile', 'getFile'):
# putFile and getFile return for each entry
# in the successful dir the size of the corresponding file
totalSize = sum(succ.values())
elif self.methodName in ('putDirectory', 'getDirectory'):
# putDirectory and getDirectory return for each dir name
# a dictionnary with the keys 'Files' and 'Size'
totalSize = sum(
val.get('Size', 0) for val in succ.values()
if isinstance(val, dict))
totalSucc = sum(
val.get('Files', 0) for val in succ.values()
if isinstance(val, dict))
oDataOperation.setValueByKey('TransferOK', len(succ))
oDataOperation.setValueByKey('TransferSize', totalSize)
oDataOperation.setValueByKey('TransferTotal', totalSucc)
oDataOperation.setValueByKey('TransferOK', totalSucc)
if callRes['Value']['Failed']:
oDataOperationFailed = copy.deepcopy(oDataOperation)
oDataOperationFailed.setValueByKey('TransferTotal',
len(failed))
oDataOperationFailed.setValueByKey('TransferOK', 0)
oDataOperationFailed.setValueByKey('TransferSize', 0)
oDataOperationFailed.setValueByKey('FinalStatus', 'Failed')
accRes = gDataStoreClient.addRegister(oDataOperationFailed)
if not accRes['OK']:
self.log.error("Could not send failed accounting report",
accRes['Message'])
accRes = gDataStoreClient.addRegister(oDataOperation)
if not accRes['OK']:
self.log.error("Could not send accounting report",
accRes['Message'])
#!/usr/bin/env python
import sys,os
from subprocess import call
from DIRAC import siteName
logFile = open('job.log', 'w')
errFile = open('job.err', 'w')
'''********************Preparation********************'''
jobID = os.environ.get('DIRACJOBID', '0')
siteName = siteName()
call(['python','checkExeEnv.py'])
result = call(['python','checkCvmfs.py','/cvmfs/cepc/ihep.ac.cn/',jobID])
if result!=0:
sys.exit(result)
call(['python','listInputSanbox.py'])
call(['python','determineMirrorDB.py',siteName])
max_q_time = 60 * 5
call(['python','determineQueue.py',siteName,max_q_time])
'''********************Execute Simulation********************'''
call(['python','tmsg.py','Generate shell script for simulation'])
call(['chmod','755','simu.sh'])
call(['python','setJobStatus.py',jobID,'CEPC_script','Mokka Simulation'])
call(['python','tmsg.py','Start simulation'])
result = call(['./simu.sh','Mokka'])
call(['python','checkSimuLog.py',result])