class BookkeepingWatchAgent(AgentModule, TransformationAgentsUtilities):
""" LHCbDIRAC only agent. A threaded agent.
"""
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
TransformationAgentsUtilities.__init__(self)
self.bkQueriesToBeChecked = Queue.Queue()
self.bkQueriesInCheck = []
self.fullUpdatePeriod = 86400
self.bkUpdateLatency = 7200
self.debug = False
self.transInThread = {}
self.pickleFile = 'BookkeepingWatchAgent.pkl'
self.chunkSize = 1000
self.pluginsWithNoRunInfo = ['LHCbStandard', 'ReplicateDataset', 'ArchiveDataset',
'LHCbMCDSTBroadcastRandom', 'ReplicateToLocalSE',
'RemoveReplicas', 'RemoveReplicasWhenProcessed',
'RemoveReplicasWithAncestors', 'ReplicateWithAncestors',
'ReduceReplicas', 'RemoveDatasetFromDisk',
'DestroyDataset', 'DestroyDatasetWhenProcessed',
'BySize', 'Standard']
self.timeLog = {}
self.fullTimeLog = {}
self.bkQueries = {}
self.transClient = None
self.bkClient = None
def initialize(self):
""" Make the necessary initializations.
The ThreadPool is created here, the _execute() method is what each thread will execute.
"""
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', self.fullUpdatePeriod)
self.bkUpdateLatency = self.am_getOption('BKUpdateLatency', self.bkUpdateLatency)
self.debug = self.am_getOption('verbose', self.debug)
self.pickleFile = os.path.join(self.am_getWorkDirectory(), self.pickleFile)
self.chunkSize = self.am_getOption('maxFilesPerChunk', self.chunkSize)
self.pluginsWithNoRunInfo = Operations().getValue('TransformationPlugins/PluginsWithNoRunInfo',
self.pluginsWithNoRunInfo)
self._logInfo('Full Update Period: %d seconds' % self.fullUpdatePeriod)
self._logInfo('BK update latency : %d seconds' % self.bkUpdateLatency)
self._logInfo('Plugins with no run info: %s' % ', '.join(self.pluginsWithNoRunInfo))
self.transClient = TransformationClient()
self.bkClient = BookkeepingClient()
try:
with open(self.pickleFile, 'r') as pf:
self.timeLog = pickle.load(pf)
self.fullTimeLog = pickle.load(pf)
self.bkQueries = pickle.load(pf)
self._logInfo("successfully loaded Log from", self.pickleFile, "initialize")
except (EOFError, IOError):
self._logInfo("failed loading Log from", self.pickleFile, "initialize")
self.timeLog = {}
self.fullTimeLog = {}
self.bkQueries = {}
maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
for i in xrange(maxNumberOfThreads):
threadPool.generateJobAndQueueIt(self._execute, [i])
gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM)
return S_OK()
@gSynchro
def __dumpLog(self):
""" dump the log in the pickle file
"""
if self.pickleFile:
try:
with open(self.pickleFile, 'w') as pf:
pickle.dump(self.timeLog, pf)
pickle.dump(self.fullTimeLog, pf)
pickle.dump(self.bkQueries, pf)
self._logVerbose("successfully dumped Log into %s" % self.pickleFile)
except IOError as e:
self._logError("fail to open %s: %s" % (self.pickleFile, e))
except pickle.PickleError as e:
self._logError("fail to dump %s: %s" % (self.pickleFile, e))
except ValueError as e:
self._logError("fail to close %s: %s" % (self.pickleFile, e))
################################################################################
def execute(self):
""" Main execution method. Just fills a list, and a queue, with BKKQueries ID.
"""
gMonitor.addMark('Iteration', 1)
# Get all the transformations
result = self.transClient.getTransformations(condDict={'Status': ['Active', 'Idle']})
if not result['OK']:
self._logError("Failed to get transformations.", result['Message'])
return S_OK()
transIDsList = [long(transDict['TransformationID']) for transDict in result['Value']]
res = self.transClient.getTransformationsWithBkQueries(transIDsList)
if not res['OK']:
self._logError("Failed to get transformations with Bk Queries.", res['Message'])
return S_OK()
transIDsWithBkQueriesList = res['Value']
_count = 0
# Process each transformation
for transID in transIDsWithBkQueriesList:
if transID in self.bkQueriesInCheck:
continue
self.bkQueriesInCheck.append(transID)
self.bkQueriesToBeChecked.put(transID)
_count += 1
self._logInfo("Out of %d transformations, %d put in thread queue" % (len(result['Value']), _count))
self.__dumpLog()
return S_OK()
def _execute(self, threadID):
""" Real executor. This is what is executed by the single threads - so do not return here! Just continue
"""
while True: # not self.bkQueriesToBeChecked.empty():
transID = None
try:
transID = self.bkQueriesToBeChecked.get()
self.transInThread[transID] = ' [Thread%d] [%s] ' % (threadID, str(transID))
startTime = time.time()
self._logInfo("Processing transformation %s." % transID, transID=transID)
res = self.transClient.getTransformation(transID, extraParams=False)
if not res['OK']:
self._logError("Failed to get transformation", res['Message'], transID=transID)
continue
transPlugin = res['Value']['Plugin']
res = self.transClient.getBookkeepingQuery(transID)
if not res['OK']:
self._logError("Failed to get BkQuery", res['Message'], transID=transID)
continue
bkQuery = res['Value']
# Determine the correct time stamp to use for this transformation
now = datetime.datetime.utcnow()
self.__timeStampForTransformation(transID, bkQuery, now)
try:
files = self.__getFiles(transID, bkQuery, now)
except RuntimeError as e:
# In case we failed a full query, we should retry full query until successful
if 'StartDate' not in bkQuery:
self.bkQueries.pop(transID, None)
self._logError("Failed to get response from the Bookkeeping: %s" % e, "", "__getFiles", transID)
continue
runDict = {}
filesMetadata = {}
# get the files metadata
for lfnChunk in breakListIntoChunks(files, self.chunkSize):
start = time.time()
res = self.bkClient.getFileMetadata(lfnChunk)
self._logVerbose("Got metadata from BK for %d files" % len(lfnChunk), transID=transID, reftime=start)
if not res['OK']:
self._logError("Failed to get BK metadata for %d files" % len(lfnChunk),
res['Message'], transID=transID)
# No need to return as we only consider files that are successful...
else:
filesMetadata.update(res['Value']['Successful'])
# There is no need to add the run information for a transformation that doesn't need it
if transPlugin not in self.pluginsWithNoRunInfo:
for lfn, metadata in filesMetadata.iteritems():
runID = metadata.get('RunNumber', None)
if isinstance(runID, (basestring, int, long)):
runDict.setdefault(int(runID), []).append(lfn)
try:
self.__addRunsMetadata(transID, runDict.keys())
except RuntimeError as e:
self._logException("Failure adding runs metadata",
method="__addRunsMetadata",
lException=e,
transID=transID)
else:
runDict[None] = filesMetadata.keys()
# Add all new files to the transformation
for runID in sorted(runDict):
lfnList = runDict[runID]
# We enter all files of a run at once, otherwise do it by chunks
lfnChunks = [lfnList] if runID else breakListIntoChunks(lfnList, self.chunkSize)
for lfnChunk in lfnChunks:
# Add the files to the transformation
self._logVerbose('Adding %d lfns for transformation' % len(lfnChunk), transID=transID)
result = self.transClient.addFilesToTransformation(transID, lfnChunk)
if not result['OK']:
self._logError("Failed to add %d lfns to transformation" % len(lfnChunk), result['Message'],
transID=transID)
return result
else:
# Handle errors
errors = {}
for lfn, error in result['Value']['Failed'].iteritems():
errors.setdefault(error, []).append(lfn)
for error, lfns in errors.iteritems():
self._logWarn("Failed to add files to transformation", error, transID=transID)
self._logVerbose("\n\t".join([''] + lfns))
# Add the metadata and RunNumber to the newly inserted files
addedLfns = [lfn for (lfn, status) in result['Value']['Successful'].iteritems() if status == 'Added']
if addedLfns:
# Add files metadata: size and file type
lfnDict = dict((lfn, {'Size': filesMetadata[lfn]['FileSize'],
'FileType': filesMetadata[lfn]['FileType']})
for lfn in addedLfns)
res = self.transClient.setParameterToTransformationFiles(transID, lfnDict)
if not res['OK']:
self._logError("Failed to set transformation files metadata", res['Message'])
return res
# Add run information if it exists
if runID:
self._logInfo("Added %d files to transformation for run %d, now including run information"
% (len(addedLfns), runID), transID=transID)
self._logVerbose("Associating %d files to run %d" % (len(addedLfns), runID), transID=transID)
res = self.transClient.addTransformationRunFiles(transID, runID, addedLfns)
if not res['OK']:
self._logError("Failed to associate %d files to run %d" % (len(addedLfns), runID),
res['Message'], transID=transID)
return res
else:
self._logInfo("Added %d files to transformation" % len(addedLfns), transID=transID)
except Exception as x: # pylint: disable=broad-except
self._logException('Exception while adding files to transformation',
lException=x,
method='_execute',
transID=transID)
finally:
self._logInfo("Processed transformation", transID=transID, reftime=startTime)
if transID in self.bkQueriesInCheck:
self.bkQueriesInCheck.remove(transID)
self.transInThread.pop(transID, None)
return S_OK()
@gSynchro
def __timeStampForTransformation(self, transID, bkQuery, now):
""" Determine the correct time stamp to use for this transformation
"""
fullTimeLog = self.fullTimeLog.setdefault(transID, now)
bkQueryLog = self.bkQueries.setdefault(transID, {})
bkQueryLog.pop('StartDate', None)
self.bkQueries[transID] = bkQuery.copy()
if transID in self.timeLog \
and bkQueryLog == bkQuery \
and (now - fullTimeLog) < datetime.timedelta(seconds=self.fullUpdatePeriod):
# If it is more than a day since the last reduced query, make a full query just in case
timeStamp = self.timeLog[transID]
delta = datetime.timedelta(seconds=self.bkUpdateLatency)
bkQuery['StartDate'] = (timeStamp - delta).strftime('%Y-%m-%d %H:%M:%S')
if 'StartDate' not in bkQuery:
self.fullTimeLog[transID] = now
def __getFiles(self, transID, bkQuery, now):
""" Perform the query to the Bookkeeping
"""
self._logInfo("Using BK query for transformation: %s" % str(bkQuery), transID=transID)
start = time.time()
result = self.bkClient.getFiles(bkQuery)
self._logVerbose("BK query time: %.2f seconds." % (time.time() - start), transID=transID)
if not result['OK']:
raise RuntimeError(result['Message'])
else:
self.__updateTimeStamp(transID, now)
if result['Value']:
self._logInfo("Obtained %d files from BK" % len(result['Value']), transID=transID)
return result['Value']
@gSynchro
def __updateTimeStamp(self, transID, now):
"""
Update time stamp for current transformation to now
"""
self.timeLog[transID] = now
def __addRunsMetadata(self, transID, runsList):
""" Add the run metadata
"""
runsInCache = self.transClient.getRunsInCache({'Name': ['TCK', 'CondDb', 'DDDB']})
if not runsInCache['OK']:
raise RuntimeError(runsInCache['Message'])
newRuns = list(set(runsList) - set(runsInCache['Value']))
if newRuns:
self._logVerbose("Associating run metadata to %d runs" % len(newRuns), transID=transID)
res = self.bkClient.getRunInformation({'RunNumber': newRuns, 'Fields': ['TCK', 'CondDb', 'DDDB']})
if not res['OK']:
raise RuntimeError(res['Message'])
else:
for run, runMeta in res['Value'].iteritems():
res = self.transClient.addRunsMetadata(run, runMeta)
if not res['OK']:
raise RuntimeError(res['Message'])
# Add run duration to the metadata
runsInCache = self.transClient.getRunsInCache({'Name': ['Duration']})
if not runsInCache['OK']:
raise RuntimeError(runsInCache['Message'])
newRuns = list(set(runsList) - set(runsInCache['Value']))
if newRuns:
self._logVerbose("Associating run duration to %d runs" % len(newRuns), transID=transID)
res = self.bkClient.getRunInformation({'RunNumber': newRuns, 'Fields': ['JobStart', 'JobEnd']})
if not res['OK']:
raise RuntimeError(res['Message'])
else:
for run, runMeta in res['Value'].iteritems():
duration = (runMeta['JobEnd'] - runMeta['JobStart']).seconds
res = self.transClient.addRunsMetadata(run, {'Duration': duration})
if not res['OK']:
raise RuntimeError(res['Message'])
def finalize(self):
""" Gracious finalization
"""
if self.bkQueriesInCheck:
self._logInfo("Wait for queue to get empty before terminating the agent (%d tasks)" % len(self.transInThread))
self.bkQueriesInCheck = []
while self.transInThread:
time.sleep(2)
self.log.info("Threads are empty, terminating the agent...")
return S_OK()
class ModuleBase(object):
""" Base class for Modules - works only within DIRAC workflows
"""
#############################################################################
def __init__(self, loggerIn=None, operationsHelperIn=None, bkClientIn=None, dm=None):
""" Initialization of module base.
"""
if loggerIn is None:
self.log = gLogger.getSubLogger('ModuleBase')
else:
self.log = loggerIn
if operationsHelperIn is None:
self.opsH = Operations()
else:
self.opsH = operationsHelperIn
if bkClientIn is None:
self.bkClient = BookkeepingClient()
else:
self.bkClient = bkClientIn
if dm is None:
self.dataManager = DataManager()
else:
self.dataManager = dm
self.requestValidator = RequestValidator()
self.production_id = ''
self.prod_job_id = ''
self.jobID = 0
self.step_number = ''
self.step_id = ''
self.fileReport = None
self.jobReport = None
self.request = None
self.workflowStatus = None
self.stepStatus = None
self.workflow_commons = None
self.step_commons = None
self.debugSE = 'CERN-DEBUG'
self.executable = 'gaudirun.py'
self.applicationName = 'Unknown'
self.applicationVersion = 'Unknown'
self.applicationLog = ''
self.applicationType = None
self.systemConfig = None
self.extraPackages = None
self.bkConfigName = None
self.BKstepID = None
self.condDBTag = None
self.DDDBTag = None
self.dqTag = None
self.CPUe = None
self.eventType = ''
self.gaudiSteps = None
self.InputData = ''
self.inputDataList = []
self.inputDataType = None
self.histoName = "Hist.root"
self.optionsFile = None
self.optionsFormat = None
self.optionsLine = None
self.extraOptionsLine = None
self.jobType = None
self.logFilePath = None
self.onlineCondDBTag = None
self.onlineDDDBTag = None
self.outputSEs = {}
self.outputDataFileMask = None
self.numberOfEvents = -1
self.maxNumberOfEvents = None
self.TCK = None
self.mcTCK = None
self.multicoreJob = None
self.multicoreStep = None
self.poolXMLCatName = 'pool_xml_catalog.xml'
self.persistency = ''
self.processingPass = None
self.runNumber = 'Unknown'
self.runTimeProjectName = None
self.runTimeProjectVersion = None
self.simDescription = ''
self.siteName = None
self.stepName = None
self.stepInputData = None
self.XMLSummary = '' # name of the file, not the object
self.stepProcPass = None
self.outputFilePrefix = ''
#############################################################################
def execute(self, version=None,
production_id=None, prod_job_id=None, wms_job_id=None,
workflowStatus=None, stepStatus=None,
wf_commons=None, step_commons=None,
step_number=None, step_id=None):
""" Function called by all super classes
"""
if version:
self.log.info('===== Executing ' + version + ' ===== ')
self.log.verbose("Executing directory for job is %s" % os.getcwd())
if production_id:
self.production_id = production_id
else:
self.production_id = self.workflow_commons['PRODUCTION_ID'] # This is a string, like '00051753'
if prod_job_id:
self.prod_job_id = prod_job_id
else:
self.prod_job_id = self.workflow_commons['JOB_ID']
if 'JOBID' in os.environ:
self.jobID = os.environ['JOBID']
if wms_job_id:
self.jobID = wms_job_id
if workflowStatus:
self.workflowStatus = workflowStatus
if stepStatus:
self.stepStatus = stepStatus
if wf_commons:
self.workflow_commons = wf_commons
if step_commons:
self.step_commons = step_commons
if step_number:
self.step_number = step_number
else:
self.step_number = self.STEP_NUMBER # pylint: disable=no-member
if step_id:
self.step_id = step_id
else:
self.step_id = '%s_%s_%s' % (self.production_id, self.prod_job_id, self.step_number)
self.siteName = siteName()
#############################################################################
def finalize(self, version=None):
""" Just finalizing
"""
self.log.flushAllMessages(0)
if version:
self.log.info('===== Terminating ' + version + ' ===== ')
#############################################################################
def setApplicationStatus(self, status, sendFlag=True):
"""Wraps around setJobApplicationStatus of state update client
"""
if not self._WMSJob():
return 0 # e.g. running locally prior to submission
if self._checkWFAndStepStatus(noPrint=True):
# The application status won't be updated in case the workflow or the step is failed already
if not isinstance(status, str):
status = str(status)
self.log.verbose('setJobApplicationStatus(%d, %s)' % (self.jobID, status))
jobStatus = self.jobReport.setApplicationStatus(status, sendFlag)
if not jobStatus['OK']:
self.log.warn(jobStatus['Message'])
#############################################################################
def setJobParameter(self, name, value, sendFlag=True):
"""Wraps around setJobParameter of state update client
"""
if not self._WMSJob():
return 0 # e.g. running locally prior to submission
self.log.verbose('setJobParameter(%d,%s,%s)' % (self.jobID, name, value))
jobParam = self.jobReport.setJobParameter(str(name), str(value), sendFlag)
if not jobParam['OK']:
self.log.warn(jobParam['Message'])
#############################################################################
def _resolveInputVariables(self):
""" By convention the module input parameters are resolved here.
"""
self.log.verbose("workflow_commons = ", self.workflow_commons)
self.log.verbose("step_commons = ", self.step_commons)
self.fileReport = self._getFileReporter()
self.jobReport = self._getJobReporter()
self.request = self._getRequestContainer()
self.__resolveInputWorkflow()
#############################################################################
def __resolveInputWorkflow(self):
""" Resolve the input variables that are in the workflow_commons
"""
self.runNumber = self.workflow_commons.get('runNumber', self.runNumber)
self.persistency = self.workflow_commons.get('persistency', self.persistency)
self.jobType = self.workflow_commons.get('JobType', self.jobType)
self.poolXMLCatName = self.workflow_commons.get('poolXMLCatName', self.poolXMLCatName)
self.InputData = self.workflow_commons.get('InputData', self.InputData)
if 'ParametricInputData' in self.workflow_commons:
pID = copy.deepcopy(self.workflow_commons['ParametricInputData'])
if pID:
if isinstance(pID, list):
pID = ';'.join(pID)
# self.InputData += ';' + pID
self.InputData = pID
self.InputData = self.InputData.rstrip(';')
if self.InputData == ';':
self.InputData = ''
self.inputDataList = [lfn.strip('LFN:') for lfn in self.InputData.split(';') if lfn]
# only required until the stripping is the same for MC / data
self.bkConfigName = self.workflow_commons.get('configName', self.bkConfigName)
self.simDescription = self.workflow_commons.get('simDescription', self.simDescription)
if 'runMetadata' in self.workflow_commons:
runMetadataDict = eval(self.workflow_commons['runMetadata'])
self.onlineDDDBTag = runMetadataDict['DDDB']
self.onlineCondDBTag = runMetadataDict['CondDb']
self.TCK = runMetadataDict['TCK']
if 'outputDataFileMask' in self.workflow_commons:
self.outputDataFileMask = self.workflow_commons['outputDataFileMask']
if not isinstance(self.outputDataFileMask, list):
self.outputDataFileMask = [i.lower().strip() for i in self.outputDataFileMask.split(';')]
self.gaudiSteps = self.workflow_commons.get('gaudiSteps', self.gaudiSteps)
if 'CPUe' in self.workflow_commons:
self.CPUe = int(round(float(self.workflow_commons['CPUe'])))
self.multicoreJob = self.workflow_commons.get('multicore', self.multicoreJob)
self.processingPass = self.workflow_commons.get('processingPass', self.processingPass)
self.logFilePath = self.workflow_commons.get('LogFilePath', self.logFilePath)
if isinstance(self.logFilePath, list):
self.logFilePath = self.logFilePath[0]
else:
if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons:
self.log.info('LogFilePath parameter not found, creating on the fly')
result = getLogPath(self.workflow_commons, self.bkClient)
if not result['OK']:
self.log.error('Could not create LogFilePath', result['Message'])
raise RuntimeError(result['Message'])
self.logFilePath = result['Value']['LogFilePath'][0]
if 'maxNumberOfEvents' in self.workflow_commons:
self.maxNumberOfEvents = int(self.workflow_commons['maxNumberOfEvents'])
self.eventType = self.workflow_commons.get('eventType', self.eventType)
self.numberOfEvents = int(self.workflow_commons.get('numberOfEvents', self.numberOfEvents))
if 'outputSEs' in self.workflow_commons:
self.outputSEs = self.workflow_commons['outputSEs']
else:
# this is here for backward compatibility
histogramSE = self.opsH.getValue('Productions/HistogramSE', 'CERN-HIST')
histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST',
'GAUSSHIST'])
self.outputSEs = dict((ht, histogramSE) for ht in histoTypes)
# for older productions we construct it based on what should be found in the steps
if 'listoutput' in self.step_commons:
listOutputStep = self.step_commons['listoutput']
for lOutput in listOutputStep:
try:
for outputDataType in lOutput['outputDataType'].split(';'):
if outputDataType:
self.outputSEs.setdefault(outputDataType.upper(), lOutput['outputDataSE'])
except KeyError:
continue
self.workflow_commons['outputSEs'] = self.outputSEs
#############################################################################
def _resolveInputStep(self):
""" Resolve the input variables for an application step
"""
prodID = self.workflow_commons.get('PRODUCTION_ID', '')
jobID = self.workflow_commons.get('JOB_ID', '')
stepInstanceNumber = self.step_commons.get('STEP_NUMBER', '')
self.stepName = self.step_commons['STEP_INSTANCE_NAME']
self.executable = self.step_commons.get('executable', self.executable)
self.applicationName = self.step_commons.get('applicationName', self.applicationName)
self.applicationVersion = self.step_commons.get('applicationVersion', self.applicationVersion)
self.BKstepID = self.step_commons.get('BKStepID', self.BKstepID)
self.stepProcPass = self.step_commons.get('StepProcPass', self.stepProcPass)
# this is only for production jobs and for application steps
if prodID and jobID and stepInstanceNumber and 'listoutput' in self.step_commons:
self.outputFilePrefix = "%s_%s_%s" % (prodID, jobID, stepInstanceNumber)
self.applicationLog = self.applicationName + '_' + self.outputFilePrefix + '.log'
self.XMLSummary = 'summary' + self.applicationName + '_' + self.outputFilePrefix + '.xml'
self.histoName = self.applicationName + '_' + self.outputFilePrefix + '.Hist.root'
for fileTypeDict in self.step_commons['listoutput']: # this is a dict like {'outputDataType': 'sim'}
# for non histo-merging prods
if 'hist' in fileTypeDict['outputDataType'].lower() and self.jobType.lower() != 'merge':
# Watch out: this assumes that:
# - 'hist' is always in the file type name
# - merging jobs won't produce histograms
# - the only merging jobs that produce output types with hist are histomerging productions
if 'outputDataName' not in fileTypeDict:
fileTypeDict['outputDataName'] = self.histoName
else:
fileTypeDict['outputDataName'] = self.outputFilePrefix + '.' + fileTypeDict['outputDataType']
else:
self.applicationLog = self.step_commons.get('applicationLog', self.applicationLog)
self.inputDataType = self.step_commons.get('inputDataType', self.inputDataType)
self.applicationType = self.step_commons.get('applicationType', self.applicationType)
self.optionsFile = self.step_commons.get('optionsFile', self.optionsFile)
self.optionsLine = self.step_commons.get('optionsLine', self.optionsLine)
self.extraOptionsLine = self.step_commons.get('extraOptionsLine', self.extraOptionsLine)
if 'runTimeProjectName' in self.step_commons:
self.runTimeProjectName = self.step_commons['runTimeProjectName']
self.runTimeProjectVersion = self.step_commons['runTimeProjectVersion']
if 'extraPackages' in self.step_commons:
self.extraPackages = self.step_commons['extraPackages']
if self.extraPackages:
if isinstance(self.extraPackages, basestring):
eps = self.extraPackages.split(';') # pylint: disable=no-member
epList = []
for ep in eps:
epList.append(tuple(ep.split('.')))
self.extraPackages = epList
stepInputData = []
if 'inputData' in self.step_commons:
if self.step_commons['inputData']:
stepInputData = self.step_commons['inputData']
elif self.InputData:
stepInputData = copy.deepcopy(self.InputData)
if stepInputData:
stepInputData = self._determineStepInputData(stepInputData, )
self.stepInputData = [sid.strip('LFN:') for sid in stepInputData]
self.optionsFormat = self.step_commons.get('optionsFormat', self.optionsFormat)
self.multicoreStep = self.step_commons.get('multiCore', self.multicoreStep)
self.systemConfig = self.step_commons.get('SystemConfig', self.systemConfig)
self.mcTCK = self.step_commons.get('mcTCK', self.mcTCK)
self.DDDBTag = self.step_commons.get('DDDBTag', self.DDDBTag)
self.condDBTag = self.step_commons.get('CondDBTag', self.condDBTag)
self.dqTag = self.step_commons.get('DQTag', self.dqTag)
# This is resolved also in __resolveInputWorkflow but can be specialized per step (e.g. LHCbJob().setApplication())
self.numberOfEvents = int(self.step_commons.get('numberOfEvents', self.numberOfEvents))
#############################################################################
def _determineOutputs(self):
""" Method that determines the correct outputs.
For merging jobs the output has normally to be the same as the input, but there might be exceptions
(like for the productions for the merging of histograms)
For the others, we use what is in the step definition
We always remove the 'HIST'(s), when present, from the list of output file types
as these are treated differently.
There is anyway also here the special case of histogram merging productions.
"""
histoTypes = self.opsH.getValue('Productions/HistogramTypes', ['HIST', 'BRUNELHIST', 'DAVINCIHIST', 'GAUSSHIST'])
stepOutputs = self.step_commons['listoutput']
stepOutputsT = [x['outputDataType'] for x in stepOutputs]
stepOutTypes = []
for fts in stepOutputsT:
for ft in fts.split(';'):
if ft and ft not in stepOutTypes:
stepOutTypes.append(ft.lower())
# corrections for Merge productions
if self.jobType.lower() in ('merge', 'histomerge'):
if len(stepOutTypes) == 1 and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']:
# If it is root/histo/ntuple merging job, we treat it almost as a stripping job
pass
else:
res = self.bkClient.getFileMetadata(self.stepInputData)
if not res['OK']:
raise RuntimeError(res['Message'])
outputTypes = set()
success = res['Value']['Successful']
for mdDict in success.values():
outputTypes.add(mdDict['FileType'])
if len(success) != len(self.stepInputData):
self.log.warn("Some inputs are not in BKK, trying to parse the file names")
for sid in set(self.stepInputData) - set(success):
# File types in the BK are upper case
fType = '.'.join(os.path.basename(sid).split('.')[1:]).upper()
outputTypes.add(fType)
outputTypes = list(outputTypes)
if len(outputTypes) > 1:
raise ValueError("Not all input files have the same type (%s)" % ','.join(outputTypes))
outputType = outputTypes[0].lower()
stepOutTypes = [outputType.lower()]
stepOutputs = [{'outputDataName': self.step_id + '.' + outputType.lower(),
'outputDataType': outputType.lower(),
'outputBKType': outputType.upper()}]
histogram = False
# first here treating the special case of root/histo/ntuple merging jobs
if self.jobType.lower() in ('merge', 'histomerge') \
and len(stepOutTypes) == 1 \
and stepOutTypes[0] in [hts.lower() for hts in histoTypes] + ['root']:
pass
else: # This is not a histogram merging production, so we remove the histograms from the step outputs
for hist in histoTypes:
try:
stepOutTypes.remove(hist)
histogram = True
except ValueError:
pass
try:
stepOutTypes.remove(hist.lower())
histogram = True
except ValueError:
pass
return stepOutputs, stepOutTypes, histogram
#############################################################################
def _getJobReporter(self):
""" just return the job reporter (object, always defined by dirac-jobexec)
"""
if 'JobReport' in self.workflow_commons:
return self.workflow_commons['JobReport']
jobReport = JobReport(self.jobID)
self.workflow_commons['JobReport'] = jobReport
return jobReport
#############################################################################
def _getFileReporter(self):
""" just return the file reporter (object)
"""
if 'FileReport' in self.workflow_commons:
return self.workflow_commons['FileReport']
fileReport = FileReport()
self.workflow_commons['FileReport'] = fileReport
return fileReport
#############################################################################
def _getRequestContainer(self):
""" just return the Request reporter (object)
"""
if 'Request' in self.workflow_commons:
return self.workflow_commons['Request']
request = Request()
self.workflow_commons['Request'] = request
return request
#############################################################################
def getCandidateFiles(self, outputList, outputLFNs, fileMask='', stepMask=''):
""" Returns list of candidate files to upload, check if some outputs are missing.
:param list outputList: list of outputs with the following structure::
[{'outputDataType': '', 'outputDataName': ''} , {...}]
:param list outputLFNs: output LFNs for the job
:param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings
:param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings.
:returns: dictionary containing type, SE and LFN for files restricted by mask
"""
fileInfo = {}
for outputFile in outputList:
if 'outputDataType' in outputFile and 'outputDataName' in outputFile:
fname = outputFile['outputDataName']
fileType = outputFile['outputDataType']
fileInfo[fname] = {'type': fileType}
else:
self.log.error('Ignoring mal-formed output data specification', str(outputFile))
for lfn in outputLFNs:
if os.path.basename(lfn).lower() in list(fi.lower() for fi in fileInfo):
try:
fileInfo[os.path.basename(lfn)]['lfn'] = lfn
self.log.verbose("Found LFN for file",
"%s -> %s" % (lfn, os.path.basename(lfn)))
except KeyError:
fileInfo[os.path.basename(lfn).lower()]['lfn'] = lfn
self.log.verbose("Found LFN for file",
"%s -> %s" % (lfn, os.path.basename(lfn).lower()))
elif os.path.basename(lfn).split('_')[-1].lower() in list(fi.lower() for fi in fileInfo):
try:
fileInfo[os.path.basename(lfn).split('_')[-1]]['lfn'] = lfn
self.log.verbose("Found LFN for file",
" %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1]))
except KeyError:
fileInfo[os.path.basename(lfn).split('_')[-1].lower()]['lfn'] = lfn
self.log.verbose("Found LFN for file",
" %s -> %s" % (lfn, os.path.basename(lfn).split('_')[-1].lower()))
else:
self.log.warn("LFN not recognized", "for LFN %s" % lfn)
# check local existance
fileList = self._checkLocalExistance(list(fileInfo.keys()))
# really horrible stuff for updating the name with what's found on the disk
# (because maybe the case is not the same as the expected)
newFileInfo = {}
for fi in fileInfo.iteritems():
for li in fileList:
if fi[0].lower() == li.lower():
newFileInfo[li] = fi[1]
# Select which files have to be uploaded: in principle all
candidateFiles = self._applyMask(newFileInfo, fileMask, stepMask)
# Sanity check all final candidate metadata keys are present
self._checkSanity(candidateFiles)
# Adding the SEs
for candidateFile in candidateFiles:
try:
fType = candidateFiles[candidateFile]['type']
for fType in [fType, fType.lower(), fType.upper(), fType.capitalize()]:
try:
wfSE = self.outputSEs[fType]
candidateFiles[candidateFile]['workflowSE'] = wfSE
break
except KeyError:
continue
except AttributeError:
break
return candidateFiles
#############################################################################
def _checkLocalExistance(self, fileList):
""" Check that the list of output files are present locally
"""
notPresentFiles = []
filesOnDisk = set(os.listdir('.'))
diff = set(fileList).difference(filesOnDisk)
if diff:
self.log.warn("Not all files found",
"set of what's on the disk: %s" % filesOnDisk)
self.log.warn("Looking for files with different case")
diffCI = set(fx.lower() for fx in fileList).difference(set(fod.lower() for fod in filesOnDisk))
if diffCI:
self.log.error("Output data not found",
"File list %s does not exist locally" % notPresentFiles)
raise os.error("Output data not found")
# now checking what's the actual filename case that's written
self.log.warn("Found files with filename with different case, returning those")
filesActuallyOnDisk = []
for fod in os.listdir('.'):
if fod.lower() in [fl.lower() for fl in fileList]:
filesActuallyOnDisk.append(fod)
return filesActuallyOnDisk
return fileList
#############################################################################
def _applyMask(self, candidateFilesIn, fileMask, stepMask):
""" Select which files have to be uploaded: in principle all
:param dict candidateFilesIn: dictionary like
{'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst',
type': 'dst'},
'00012345_00012345_2.digi': {'type': 'digi'}}
:param str fileMask: the output file extensions to restrict the outputs to. Can also be a list of strings
:param str stepMask: the step ID to restrict the outputs to. Can also be a list of strings.
:returns: a dict like the one in candidateFilesIn
"""
candidateFiles = copy.deepcopy(candidateFilesIn)
if fileMask and not isinstance(fileMask, list):
fileMask = [fileMask]
if isinstance(stepMask, int):
stepMask = str(stepMask)
if stepMask and not isinstance(stepMask, list):
stepMask = [stepMask]
if fileMask and fileMask != ['']:
for fileName, metadata in list(candidateFiles.iteritems()):
if metadata['type'].lower() not in [fm.lower() for fm in fileMask]:
del candidateFiles[fileName]
self.log.info('Output file %s was produced but will not be treated (fileMask is %s)' % (fileName,
', '.join(fileMask)))
else:
self.log.info('No outputDataFileMask provided, the files with all the extensions will be considered')
if stepMask and stepMask != ['']:
for fileName, metadata in list(candidateFiles.iteritems()):
if fileName.lower().replace(metadata['type'].lower(), '').split('_')[-1].split('.')[0] not in stepMask:
del candidateFiles[fileName]
self.log.info('Output file %s was produced but will not be treated (stepMask is %s)' % (fileName,
', '.join(stepMask)))
else:
self.log.info('No outputDataStep provided, the files output of all the steps will be considered')
return candidateFiles
#############################################################################
def _checkSanity(self, candidateFiles):
""" Sanity check all final candidate metadata keys are present
:param dict candidateFiles: dictionary like
{'00012345_00012345_4.dst': {'lfn': '/lhcb/MC/2010/DST/123/123_45_4.dst',
type': 'dst'},
'00012345_00012345_2.digi': {'type': 'digi'}}
:returns: None or raises ValueError
"""
notPresentKeys = []
mandatoryKeys = ['type', 'lfn'] # filedict is used for requests
for fileName, metadata in candidateFiles.iteritems():
for key in mandatoryKeys:
if key not in metadata:
notPresentKeys.append((fileName, key))
if notPresentKeys:
for fileName_keys in notPresentKeys:
self.log.error("File %s has missing %s" % (fileName_keys[0], fileName_keys[1]))
raise ValueError("Missing requested fileName keys")
#############################################################################
def getFileMetadata(self, candidateFiles):
""" Returns the candidate file dictionary with associated metadata.
The input candidate files dictionary has the structure:
{'foo_1.txt': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/foo_1.txt',
'type': 'txt',
'workflowSE': SE1},
'bar_2.py': {'lfn': '/lhcb/MC/2010/DST/00012345/0001/bar_2.py',
'type': 'py',
'workflowSE': 'SE2'},
}
this also assumes the files are in the current working directory.
"""
# Retrieve the POOL File GUID(s) for any final output files
self.log.info('Will search for POOL GUIDs for: %s' % (', '.join(candidateFiles.keys())))
pfnGUID = getGUID(candidateFiles.keys())
if not pfnGUID['OK']:
self.log.error('''PoolXMLFile failed to determine POOL GUID(s) for output file list,
these will be generated by the DataManager''', pfnGUID['Message'])
for fileName in candidateFiles.keys():
candidateFiles[fileName]['guid'] = ''
elif pfnGUID['generated']:
self.log.warn('PoolXMLFile generated GUID(s) for the following files ', ', '.join(pfnGUID['generated']))
else:
self.log.info('GUIDs found for all specified POOL files: %s' % (', '.join(candidateFiles.keys())))
for pfn, guid in pfnGUID['Value'].iteritems():
candidateFiles[pfn]['guid'] = guid
# Get all additional metadata about the file necessary for requests
final = {}
for fileName, metadata in candidateFiles.iteritems():
fileDict = {}
fileDict['LFN'] = metadata['lfn']
fileDict['Size'] = os.path.getsize(fileName)
fileDict['Checksum'] = fileAdler(fileName)
fileDict['ChecksumType'] = 'ADLER32'
fileDict['GUID'] = metadata['guid']
fileDict['Status'] = 'Waiting'
final[fileName] = metadata
final[fileName]['filedict'] = fileDict
final[fileName]['localpath'] = '%s/%s' % (os.getcwd(), fileName)
# Sanity check all final candidate metadata keys are present (return S_ERROR if not)
mandatoryKeys = ['guid', 'filedict'] # filedict is used for requests (this method adds guid and filedict)
for fileName, metadata in final.iteritems():
for key in mandatoryKeys:
if key not in metadata:
raise RuntimeError("File %s has missing %s" % (fileName, key))
return final
#############################################################################
def _determineStepInputData(self, inputData):
""" determine the input data for the step
"""
if inputData == 'previousStep':
stepIndex = self.gaudiSteps.index(self.stepName)
previousStep = self.gaudiSteps[stepIndex - 1]
stepInputData = []
for outputF in self.workflow_commons['outputList']:
try:
if outputF['stepName'] == previousStep and outputF['outputBKType'].lower() == self.inputDataType.lower():
stepInputData.append(outputF['outputDataName'])
except KeyError:
raise RuntimeError("Can't find output of step %s" % previousStep)
return stepInputData
return [x.strip('LFN:') for x in inputData.split(';')]
#############################################################################
def _manageAppOutput(self, outputs):
""" Calls self._findOutputs to find what's produced, then creates the LFNs
outputs, as called here, is created starting from step_commons['listoutput'],
but enriched with at least the outputDataName.
example of outputs:
[{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST',
'outputDataName': '00012345_00012345_2.BHADRON.DST'},
{'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST',
'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}]
:params list outputs: list of dicts of step output files descriptions
"""
if not outputs:
self.log.warn('Step outputs are not defined (normal user jobs. Not normal in productions and SAM jobs)')
return
else:
finalOutputs, _bkFileTypes = self._findOutputs(outputs)
self.log.info('Final step outputs are: %s' % (finalOutputs))
self.step_commons['listoutput'] = finalOutputs
if 'outputList' in self.workflow_commons:
for outFile in finalOutputs:
if outFile not in self.workflow_commons['outputList']:
self.workflow_commons['outputList'].append(outFile)
else:
self.workflow_commons['outputList'] = finalOutputs
if 'PRODUCTION_ID' and 'JOB_ID' and 'configVersion' and 'configName' in self.workflow_commons:
self.log.info('Attempting to recreate the production output LFNs...')
result = constructProductionLFNs(self.workflow_commons, self.bkClient)
if not result['OK']:
raise IOError("Could not create production LFNs: %s" % result['Message'])
self.workflow_commons['BookkeepingLFNs'] = result['Value']['BookkeepingLFNs']
self.workflow_commons['LogFilePath'] = result['Value']['LogFilePath']
self.workflow_commons['ProductionOutputData'] = result['Value']['ProductionOutputData']
#############################################################################
def _findOutputs(self, stepOutput):
""" Find which outputs of those in stepOutput (what are expected to be produced) are effectively produced.
stepOutput, as called here, is created starting from step_commons['listoutput']
example of stepOutput:
[{'outputDataType': 'bhadron.dst', 'outputBKType': 'BHADRON.DST',
'outputDataName': '00012345_00012345_2.BHADRON.DST'},
{'outputDataType': 'calibration.dst','outputDataType': 'CALIBRATION.DST',
'outputDataName': '00012345_00012345_2.CALIBRATION.DST'}]
:params list stepOutput: list of dicts of step output files descriptions
:returns: list, list
"""
bkFileTypes = [] # uppercase list of file types
finalOutputs = [] # list of dicts of what's found on the local disk
filesFound = []
for output in stepOutput:
found = False
fileOnDisk = None
for fileOnDisk in os.listdir('.'):
if output['outputDataName'].lower() == fileOnDisk.lower():
found = True
break
if found and fileOnDisk:
self.log.info('Found output file (case is not considered)',
'%s matching %s ' % (fileOnDisk, output['outputDataName']))
output['outputDataName'] = fileOnDisk
filesFound.append(output)
else:
self.log.error('Output not found', output['outputDataName'])
raise IOError("OutputData not found")
for fileFound in filesFound:
bkFileTypes.append(fileFound['outputDataType'].upper())
finalOutputs.append({'outputDataName': fileFound['outputDataName'],
'outputDataType': fileFound['outputDataType'].lower(),
'outputBKType': fileFound['outputDataType'].upper(),
'stepName': self.stepName})
return (finalOutputs, bkFileTypes)
#############################################################################
def _WMSJob(self):
""" Check if this job is running via WMS
"""
return True if self.jobID else False
#############################################################################
def _enableModule(self):
""" Enable module if it's running via WMS
"""
if not self._WMSJob():
self.log.info('No WMS JobID found, disabling module via control flag')
return False
self.log.verbose('Found WMS JobID', self.jobID)
return True
#############################################################################
def _checkWFAndStepStatus(self, noPrint=False):
""" Check the WF and Step status
"""
if not self.workflowStatus['OK'] or not self.stepStatus['OK']:
if not noPrint:
self.log.info('Skip this module, failure detected in a previous step')
self.log.info('Workflow status', self.workflowStatus)
self.log.info('Step Status', self.stepStatus)
return False
else:
return True
#############################################################################
def _disableWatchdogCPUCheck(self):
""" just writes a file to disable the watchdog
"""
self.log.info("Creating DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK in order to disable the Watchdog")
with open('DISABLE_WATCHDOG_CPU_WALLCLOCK_CHECK', 'w') as fopen:
fopen.write('%s' % time.asctime())
#############################################################################
def generateFailoverFile(self):
""" Retrieve the accumulated reporting request, and produce a JSON file that is consumed by the JobWrapper
"""
reportRequest = None
result = self.jobReport.generateForwardDISET()
if not result['OK']:
self.log.warn("Could not generate Operation for job report",
"with result:\n%s" % (result))
else:
reportRequest = result['Value']
if reportRequest:
self.log.info("Populating request with job report information")
self.request.addOperation(reportRequest)
if len(self.request):
try:
optimized = self.request.optimize()
except AttributeError:
optimized = {'OK': True}
if not optimized['OK']:
self.log.error("Could not optimize", optimized['Message'])
self.log.error("Not failing the job because of that, keep going")
isValid = self.requestValidator.validate(self.request)
if not isValid['OK']:
raise RuntimeError("Failover request is not valid: %s" % isValid['Message'])
else:
requestJSON = self.request.toJSON()
if requestJSON['OK']:
self.log.info("Creating failover request for deferred operations",
"for job %d" % self.jobID)
request_string = str(requestJSON['Value'])
self.log.debug(request_string)
# Write out the request string
fname = '%s_%s_request.json' % (self.production_id, self.prod_job_id)
with open(fname, 'w') as jsonFile:
jsonFile.write(request_string)
self.log.info("Created file containing failover request", fname)
result = self.request.getDigest()
if result['OK']:
self.log.info("Digest of the request", result['Value'])
else:
self.log.warn("No digest? That's not sooo important",
"anyway: %s" % result['Message'])
else:
raise RuntimeError(requestJSON['Message'])
accountingReport = None
if 'AccountingReport' in self.workflow_commons:
accountingReport = self.workflow_commons['AccountingReport']
if accountingReport:
result = accountingReport.commit()
if not result['OK']:
self.log.error("!!! Both accounting and RequestDB are down? !!!")
self.log.error("accountingReport result: %s" % result['Message'])
self.log.error("Anyway, the job won't fail for this reason, because this is \"just\" the accounting report")
#############################################################################
def setBKRegistrationRequest(self, lfn, error='',
metaData={'Checksum': 'justSomething',
'ChecksumType': 'ADLER32',
'GUID': 'aGUID'}):
""" Set a BK registration request for changing the replica flag.
Uses the global request object (self.request).
"""
if error:
self.log.info('BK registration for %s failed with message: "%s" setting failover request' % (lfn, error))
else:
self.log.info('Setting BK registration request for %s' % (lfn))
regFile = Operation()
regFile.Type = 'RegisterFile'
regFile.Catalog = 'BookkeepingDB'
bkFile = File()
bkFile.LFN = lfn
# this should NOT be needed... but RMS complains!
bkFile.PFN = lfn
bkFile.GUID = metaData['GUID']
bkFile.Checksum = metaData['Checksum']
bkFile.ChecksumType = metaData['ChecksumType']
regFile.addFile(bkFile)
res = self.request.addOperation(regFile)
if not res['OK']:
raise RuntimeError(res['Message'])
#############################################################################
def createProdConfFile(self, stepOutputTypes, histogram, runNumberGauss, firstEventNumberGauss):
""" Utility that creates a ProdConf file, used mostly as input for gaudirun jobs
"""
# Creating ProdConf file
prodConfFileName = 'prodConf_%s_%s_%s_%s.py' % (self.applicationName,
self.production_id,
self.prod_job_id,
self.step_number)
optionsDict = {}
optionsDict['Application'] = self.applicationName
optionsDict['AppVersion'] = self.applicationVersion
if self.optionsFormat:
optionsDict['OptionFormat'] = self.optionsFormat
if self.stepInputData:
optionsDict['InputFiles'] = ['LFN:' + sid for sid in self.stepInputData]
else:
if self.applicationName.lower() != "gauss":
raise RuntimeError("No MC, but no input data")
if self.outputFilePrefix:
optionsDict['OutputFilePrefix'] = self.outputFilePrefix
optionsDict['OutputFileTypes'] = stepOutputTypes
optionsDict['XMLSummaryFile'] = self.XMLSummary
optionsDict['XMLFileCatalog'] = self.poolXMLCatName
if histogram:
optionsDict['HistogramFile'] = self.histoName
if self.DDDBTag:
if self.DDDBTag.lower() == 'online':
try:
optionsDict['DDDBTag'] = self.onlineDDDBTag
self.log.debug('Set the online DDDB tag')
except NameError as e:
self.log.error('Could not find online DDDb Tag', e)
raise RuntimeError("Could not find online DDDb Tag")
else:
optionsDict['DDDBTag'] = self.DDDBTag
if self.condDBTag:
if self.condDBTag.lower() == 'online':
optionsDict['CondDBTag'] = self.onlineCondDBTag
self.log.debug('Set the online CondDB tag')
else:
optionsDict['CondDBTag'] = self.condDBTag
if self.dqTag:
optionsDict['DQTag'] = self.dqTag
if self.applicationName.lower() == 'gauss':
if self.CPUe and self.maxNumberOfEvents and self.numberOfEvents <= 0:
# Here we set maxCPUTime to 24 hours, which seems reasonable
eventsToProduce = getEventsToProduce(self.CPUe,
maxNumberOfEvents=self.maxNumberOfEvents,
jobMaxCPUTime=86400)
else:
eventsToProduce = self.numberOfEvents
else:
eventsToProduce = self.numberOfEvents
optionsDict['NOfEvents'] = eventsToProduce
if runNumberGauss:
optionsDict['RunNumber'] = runNumberGauss
if self.runNumber:
if self.runNumber not in ('Unknown', 'Multiple'):
optionsDict['RunNumber'] = self.runNumber
if firstEventNumberGauss:
optionsDict['FirstEventNumber'] = firstEventNumberGauss
# TCK: can't have both set!
if self.TCK and self.mcTCK:
raise RuntimeError("%s step: TCK set in step, and should't be!" % self.applicationName)
if self.TCK or self.mcTCK:
optionsDict['TCK'] = self.TCK if self.TCK else self.mcTCK
if self.processingPass:
optionsDict['ProcessingPass'] = self.processingPass
prodConfFile = ProdConf(prodConfFileName)
self.log.debug(optionsDict)
prodConfFile.putOptionsIn(optionsDict)
return prodConfFileName
#############################################################################
# properties
def set_jobID(self, value):
if isinstance(value, str):
if value:
value = int(value)
else:
value = 0
self._jobID = value
def get_jobID(self):
return self._jobID
jobID = property(get_jobID, set_jobID)
def setInputData(self,
lfns,
bkClient=None,
runNumber=None,
persistencyType=None):
""" Add the input data and the run number, if available
"""
if not lfns:
self.log.warn(
"no lfns passed in setInputData, was that intentional?")
return S_OK("Nothing to do")
res = Job.setInputData(self, lfns)
if not res['OK']:
return res
if not runNumber or not persistencyType:
if not bkClient:
bkClient = BookkeepingClient()
if not runNumber:
res = bkClient.getFileMetadata(lfns)
if not res['OK']:
return res
runNumbers = []
for fileMeta in res['Value']['Successful'].itervalues():
try:
if fileMeta['RunNumber'] not in runNumbers and fileMeta[
'RunNumber'] is not None:
runNumbers.append(fileMeta['RunNumber'])
except KeyError:
continue
if len(runNumbers) > 1:
runNumber = 'Multiple'
elif len(runNumbers) == 1:
runNumber = str(runNumbers[0])
else:
runNumber = 'Unknown'
if not persistencyType:
res = bkClient.getFileTypeVersion(lfns)
if not res['OK']:
return res
typeVersions = res['Value']
if not typeVersions:
self.log.verbose('The requested files do not exist in the BKK')
typeVersion = ''
else:
self.log.verbose('Found file types %s for LFNs: %s' %
(typeVersions.values(), typeVersions.keys()))
typeVersionsList = list(set(typeVersions.values()))
if len(typeVersionsList) == 1:
typeVersion = typeVersionsList[0]
else:
typeVersion = ''
self._addParameter(self.workflow, 'runNumber', 'JDL', runNumber,
'Input run number')
self._addParameter(self.workflow, 'persistency', 'String', typeVersion,
'Persistency type of the inputs')
return S_OK()
sep = '=====================================\n'
if verbose:
gLogger.always('For %s, SEs: %s' % (site, str(seList)))
pbFound = False
res = dm.getReplicas(inputData)
if not res['OK']:
gLogger.always(
"Error getting replicas for %d files" % len(inputData),
res['Message'])
continue
replicas = res['Value']['Successful']
notInFC = res['Value']['Failed']
if notInFC:
# Check if files has replica flag in the FC, If not ignore the problem
res = bk.getFileMetadata(notInFC.keys())
if not res['OK']:
gLogger.always(
'Error getting BK metadata for %d files' %
len(notInFC), res['Message'])
continue
metadata = res['Value']['Successful']
notInFC = [
lfn for lfn in notInFC
if metadata.get(lfn, {}).get('GotReplica') == 'Yes'
]
if notInFC:
pbFound = True
prettyMsg('not in the FC but in BK', notInFC)
notFoundReplicas = replicas.keys()
missingReplicas = []
class fakeClient:
def __init__(self, trans, transID, lfns, asIfProd):
self.trans = trans
self.transID = transID
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
self.transClient = TransformationClient()
from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient
self.bk = BookkeepingClient()
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
self.dm = DataManager()
self.asIfProd = asIfProd
(self.transFiles, self.transReplicas) = self.prepareForPlugin(lfns)
def addFilesToTransformation(self, transID, lfns):
return S_OK({
'Failed': {},
'Successful': dict([(lfn, 'Added') for lfn in lfns])
})
def getTransformation(self, transID, extraParams=False):
if transID == self.transID and self.asIfProd:
transID = self.asIfProd
if transID != self.transID:
return self.transClient.getTransformation(transID)
res = self.trans.getType()
return DIRAC.S_OK({'Type': res['Value']})
def getReplicas(self):
return self.transReplicas
def getFiles(self):
return self.transFiles
def getCounters(self, table, attrList, condDict):
if condDict['TransformationID'] == self.transID and self.asIfProd:
condDict['TransformationID'] = self.asIfProd
if condDict['TransformationID'] != self.transID:
return self.transClient.getCounters(table, attrList, condDict)
possibleTargets = [
'CERN-RAW', 'CNAF-RAW', 'GRIDKA-RAW', 'IN2P3-RAW', 'SARA-RAW',
'PIC-RAW', 'RAL-RAW', 'RRCKI-RAW'
]
counters = []
for se in possibleTargets:
counters.append(({'UsedSE': se}, 0))
return DIRAC.S_OK(counters)
def getBookkeepingQuery(self, transID):
if transID == self.transID and self.asIfProd:
return self.transClient.getBookkeepingQuery(asIfProd)
return self.trans.getBkQuery()
def insertTransformationRun(self, transID, runID, xx):
return DIRAC.S_OK()
def getTransformationRuns(self, condDict):
if condDict['TransformationID'] == self.transID and self.asIfProd:
condDict['TransformationID'] = self.asIfProd
if condDict['TransformationID'] == self.transID:
transRuns = []
runs = condDict.get('RunNumber', [])
if not runs and self.transFiles:
res = self.bk.getFileMetadata(
[fileDict['LFN'] for fileDict in self.transFiles])
if not res['OK']:
return res
runs = list(
set(meta['RunNumber']
for meta in res['Value']['Successful'].itervalues()))
for run in runs:
transRuns.append({
'RunNumber': run,
'Status': "Active",
"SelectedSite": None
})
return DIRAC.S_OK(transRuns)
else:
return self.transClient.getTransformationRuns(condDict)
def getTransformationFiles(self, condDict=None):
if condDict.get('TransformationID') == self.transID and self.asIfProd:
condDict['TransformationID'] = self.asIfProd
if condDict.get('TransformationID') == self.transID:
transFiles = []
if 'Status' in condDict and 'Unused' not in condDict['Status']:
return DIRAC.S_OK(transFiles)
runs = None
if 'RunNumber' in condDict:
runs = condDict['RunNumber']
if not isinstance(runs, list):
runs = [runs]
for fileDict in self.transFiles:
if not runs or fileDict['RunNumber'] in runs:
transFiles.append({
'LFN': fileDict['LFN'],
'Status': 'Unused',
'RunNumber': fileDict['RunNumber']
})
return DIRAC.S_OK(transFiles)
else:
return self.transClient.getTransformationFiles(condDict=condDict)
def setParameterToTransformationFiles(self, transID, lfnDict):
"""
Update the transFiles with some parameters
"""
if transID == self.transID:
for fileDict in self.transFiles:
fileDict.update(lfnDict.get(fileDict['LFN'], {}))
return S_OK()
else:
return self.transClient.setParameterToTransformationFiles(
transID, lfnDict)
def getTransformationFilesCount(self, transID, field, selection=None):
if selection is None:
selection = {}
if transID == self.transID or selection.get(
'TransformationID') == self.transID:
runs = selection.get('RunNumber')
if runs and not isinstance(runs, list):
runs = [runs]
if field == 'Status':
counters = {'Unused': 0}
for fileDict in self.transFiles:
if not runs or fileDict['RunNumber'] in runs:
counters['Unused'] += 1
elif field == 'RunNumber':
counters = {}
for fileDict in self.transFiles:
runID = fileDict['RunNumber']
if not runs or runID in runs:
counters.setdefault(runID, 0)
counters[runID] += 1
else:
return DIRAC.S_ERROR('Not implemented for field ' + field)
counters['Total'] = sum(count for count in counters.itervalues())
return DIRAC.S_OK(counters)
else:
return self.transClient.getTransformationFilesCount(
transID, field, selection=selection)
def getTransformationRunStats(self, transIDs):
counters = {}
for transID in transIDs:
if transID == self.transID:
for fileDict in self.transFiles:
runID = fileDict['RunNumber']
counters[transID][runID]['Unused'] = counters.setdefault(
transID, {}).setdefault(runID, {}).setdefault(
'Unused', 0) + 1
for runID in counters[transID]:
counters[transID][runID]['Total'] = counters[transID][
runID]['Unused']
else:
res = self.transClient.getTransformationRunStats(transIDs)
if res['OK']:
counters.update(res['Value'])
else:
return res
return DIRAC.S_OK(counters)
def addRunsMetadata(self, runID, val):
return self.transClient.addRunsMetadata(runID, val)
def getRunsMetadata(self, runID):
return self.transClient.getRunsMetadata(runID)
def setTransformationRunStatus(self, transID, runID, status):
return DIRAC.S_OK()
def setTransformationRunsSite(self, transID, runID, site):
return DIRAC.S_OK()
def setFileStatusForTransformation(self, transID, status, lfns):
return DIRAC.S_OK()
def addTransformationRunFiles(self, transID, run, lfns):
return DIRAC.S_OK()
def setDestinationForRun(self, runID, site):
return DIRAC.S_OK()
def getDestinationForRun(self, runID):
return self.transClient.getDestinationForRun(runID)
def prepareForPlugin(self, lfns):
import time
print "Preparing the plugin input data (%d files)" % len(lfns)
type = self.trans.getType()['Value']
if not lfns:
return (None, None)
res = self.bk.getFileMetadata(lfns)
if res['OK']:
files = []
for lfn, metadata in res['Value']['Successful'].iteritems():
runID = metadata.get('RunNumber', 0)
runDict = {"RunNumber": runID, "LFN": lfn}
files.append(runDict)
else:
print "Error getting BK metadata", res['Message']
return ([], {})
replicas = {}
startTime = time.time()
from DIRAC.Core.Utilities.List import breakListIntoChunks
for lfnChunk in breakListIntoChunks(lfns, 200):
# print lfnChunk
if type.lower() in ("replication", "removal"):
res = self.dm.getReplicas(lfnChunk, getUrl=False)
else:
res = self.dm.getReplicasForJobs(lfnChunk, getUrl=False)
# print res
if res['OK']:
for lfn, ses in res['Value']['Successful'].iteritems():
if ses:
replicas[lfn] = sorted(ses)
else:
print "Error getting replicas of %d files:" % len(
lfns), res['Message']
print "Obtained replicas of %d files in %.3f seconds" % (
len(lfns), time.time() - startTime)
return (files, replicas)
if 'BadPFN' not in problematicFiles[lfn]:
problematicFiles[lfn]['BadPFN'] = []
problematicFiles[lfn]['BadPFN'].append(se)
elif lfn in res['Value']['Successful']:
if verbose:
fp.write("Replica is ok\n")
fp.flush()
BkkChecks = False
print(
"-------- Checks LFC -> Bkk ---------------------------------------------------------"
)
fp.write(
"-------- Checks LFC -> Bkk ---------------------------------------------------------\n"
)
res = bkClient.getFileMetadata(LFNsInLFC)
if res['OK']:
BkkChecks = True
metadata = res['Value']['Successful']
missingLFNs = [
lfn for lfn in LFNsInLFC
if metadata.get(lfn, {}).get('GotReplica') == None
]
noFlagLFNs = [
lfn for lfn in LFNsInLFC
if metadata.get(lfn, {}).get('GotReplica') == 'No'
]
okLFNs = [
lfn for lfn in LFNsInLFC
if metadata.get(lfn, {}).get('GotReplica') == 'Yes'
]
class DiracLHCb(Dirac):
#############################################################################
def __init__(self, withRepo=False, repoLocation='', operationsHelperIn=None):
"""Internal initialization of the DIRAC API.
"""
super(DiracLHCb, self).__init__(withRepo=withRepo, repoLocation=repoLocation)
self.tier1s = []
if not operationsHelperIn:
self.opsH = Operations()
else:
self.opsH = operationsHelperIn
self._bkQueryTemplate = {'SimulationConditions': 'All',
'DataTakingConditions': 'All',
'ProcessingPass': '******',
'FileType': 'All',
'EventType': 'All',
'ConfigName': 'All',
'ConfigVersion': 'All',
'Production': 0,
'StartRun': 0,
'EndRun': 0,
'DataQuality': 'All',
'Visible': 'Yes'}
self._bkClient = BookkeepingClient() # to expose all BK client methods indirectly
#############################################################################
def addRootFile(self, lfn, fullPath, diracSE, printOutput=False):
""" Add a Root file to Grid storage, an attempt is made to retrieve the
POOL GUID of the file prior to upload.
Example Usage:
>>> print dirac.addFile('/lhcb/user/p/paterson/myRootFile.tar.gz','myFile.tar.gz','CERN-USER')
{'OK': True, 'Value':{'Failed': {},
'Successful': {'/lhcb/user/p/paterson/test/myRootFile.tar.gz': {'put': 64.246301889419556,
'register': 1.1102778911590576}}}}
@param lfn: Logical File Name (LFN)
@type lfn: string
@param diracSE: DIRAC SE name e.g. CERN-USER
@type diracSE: strin
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
return super(DiracLHCb, self).addFile(lfn, fullPath, diracSE,
fileGuid=makeGuid(fullPath)[fullPath],
printOutput=printOutput)
def addFile(self, lfn, fullPath, diracSE, printOutput=False): # pylint: disable=arguments-differ
""" Copy of addRootFile
"""
return super(DiracLHCb, self).addFile(lfn, fullPath, diracSE,
fileGuid=makeGuid(fullPath)[fullPath],
printOutput=printOutput)
def getBKAncestors(self, lfns, depth=1, replica=True):
""" This function allows to retrieve ancestor files from the Bookkeeping.
Example Usage:
>>> dirac.getBKAncestors('/lhcb/data/2009/DST/00005727/0000/00005727_00000042_1.dst',2)
{'OK': True, 'Value': ['/lhcb/data/2009/DST/00005727/0000/00005727_00000042_1.dst',
'/lhcb/data/2009/RAW/FULL/LHCb/COLLISION09/63807/063807_0000000004.raw']}
@param lfn: Logical File Name (LFN)
@type lfn: string or list
@param depth: Ancestor depth
@type depth: integer
"""
result = self._bkClient.getFileAncestors(lfns, depth, replica=replica)
if not result['OK']:
self.log.error('Could not get ancestors', result['Message'])
return result
ancestors = set(x['FileName'] for ancestors in result['Value']['Successful'].itervalues() for x in ancestors)
return S_OK(lfns + list(ancestors))
#############################################################################
def bkQueryRunsByDate(self, bkPath, startDate, endDate, dqFlag='All', selection='Runs'):
""" This function allows to create and perform a BK query given a supplied
BK path. The following BK path convention is expected:
/<ConfigurationName>/<Configuration Version>/<Condition Description><Processing Pass>/<Event Type>/<File Type>
so an example for 2016 collisions data would be:
/LHCb/Collision09//LHCb/Collision16/Beam6500GeV-VeloClosed-MagDown/Real Data/Reco16/Stripping26/90000000/EW.DST
The startDate and endDate must be specified as yyyy-mm-dd.
Runs can be selected based on their status e.g. the selection parameter
has the following possible attributes:
- Runs - data for all runs in the range are queried (default)
- ProcessedRuns - data is retrieved for runs that are processed
- NotProcessed - data is retrieved for runs that are not yet processed.
Example Usage:
>>> dirac.bkQueryRunsByDate('/LHCb/Collision16//Real Data/90000000/RAW',
'2016-08-20','2016-08-22',dqFlag='OK',selection='Runs')
{'OK': True, 'Value': [<LFN1>,<LFN2>]}
dirac.bkQueryRunsByDate('/LHCb/Collision16/Beam6500GeV-VeloClosed-MagDown/Real'
'Data/Reco16/Stripping26/90000000/EW.DST',
'2016-08-20','2016-08-22',dqFlag='OK',selection='Runs')
@param bkPath: BK path as described above
@type bkPath: string
@param dqFlag: Optional Data Quality flag
@type dqFlag: string
@param startDate: Start date yyyy-mm-dd
@param startDate: string
@param endDate: End date yyyy-mm-dd
@param endDate: string
@param selection: Either Runs, ProcessedRuns or NotProcessed
@param selection: string
@return: S_OK,S_ERROR
"""
runSelection = ['Runs', 'ProcessedRuns', 'NotProcessed']
if selection not in runSelection:
return S_ERROR('Expected one of %s not "%s" for selection' % (', '.join(runSelection), selection))
if not isinstance(bkPath, str):
return S_ERROR('Expected string for bkPath')
# remove any double slashes, spaces must be preserved
# remove any empty components from leading and trailing slashes
bkQuery = BKQuery().buildBKQuery(bkPath)
if not bkQuery:
return S_ERROR(
'Please provide a BK path: '
'/<ConfigurationName>/<Configuration Version>/<Condition Description>/<Processing Pass>'
'/<Event Type>/<File Type>')
if not startDate or not endDate:
return S_ERROR('Expected both start and end dates to be defined in format: yyyy-mm-dd')
if not isinstance(startDate, str) or not isinstance(endDate, str):
return S_ERROR('Expected yyyy-mm-dd string for start and end dates')
if not len(startDate.split('-')) == 3 or not len(endDate.split('-')) == 3:
return S_ERROR('Expected yyyy-mm-dd string for start and end dates')
start = time.time()
result = self._bkClient.getRunsForAGivenPeriod({'StartDate': startDate, 'EndDate': endDate})
rtime = time.time() - start
self.log.info('BK query time: %.2f sec' % rtime)
if not result['OK']:
self.log.info('Could not get runs with given dates from BK with result: "%s"' % result)
return result
if not result['Value']:
self.log.info('No runs selected from BK for specified dates')
return result
if selection not in result['Value']:
return S_ERROR('No %s runs for specified dates' % (selection))
runs = result['Value'][selection]
self.log.info('Found the following %s runs:\n%s' % (len(runs), ', '.join([str(i) for i in runs])))
# temporary until we can query for a discrete list of runs
selectedData = []
for run in runs:
query = bkQuery.copy()
query['StartRun'] = run
query['EndRun'] = run
query['CheckRunStatus'] = True if selection in ['ProcessedRuns', 'NotProcessed'] else False
if dqFlag:
check = self.__checkDQFlags(dqFlag)
if not check['OK']:
return check
dqFlag = check['Value']
query['DataQuality'] = dqFlag
start = time.time()
result = self._bkClient.getVisibleFilesWithMetadata(query)
rtime = time.time() - start
self.log.info('BK query time: %.2f sec' % rtime)
self.log.verbose(result)
if not result['OK']:
return result
self.log.info('Selected %s files for run %s' % (len(result['Value']), run))
if result['Value']['LFNs']:
selectedData += result['Value']['LFNs'].keys()
self.log.info('Total files selected = %s' % (len(selectedData)))
return S_OK(selectedData)
#############################################################################
def bkQueryRun(self, bkPath, dqFlag='All'):
""" This function allows to create and perform a BK query given a supplied
BK path. The following BK path convention is expected:
/<Run Number>/<Processing Pass>/<Event Type>/<File Type>
so an example for 2009 collisions data would be:
/63566/Real Data + RecoToDST-07/90000000/DST
In addition users can specify a range of runs using the following convention:
/<Run Number 1> - <Run Number 2>/<Processing Pass>/<Event Type>/<File Type>
so extending the above example this would look like:
/63566-63600/Real Data + RecoToDST-07/90000000/DST
Example Usage:
>>> dirac.bkQueryRun('/63566/Real Data/RecoToDST-07/90000000/DST')
{'OK':True,'Value': ['/lhcb/data/2009/DST/00005842/0000/00005842_00000008_1.dst']}
@param bkPath: BK path as described above
@type bkPath: string
@param dqFlag: Optional Data Quality flag
@type dqFlag: string
@return: S_OK,S_ERROR
"""
if not isinstance(bkPath, str):
return S_ERROR('Expected string for bkPath')
# remove any double slashes, spaces must be preserved
# remove any empty components from leading and trailing slashes
bkPath = translateBKPath(bkPath, procPassID=1)
if not len(bkPath) == 4:
return S_ERROR('Expected 4 components to the BK path: /<Run Number>/<Processing Pass>/<Event Type>/<File Type>')
runNumberString = bkPath[0].replace('--', '-').replace(' ', '')
startRun = 0
endRun = 0
if '-' in runNumberString:
runs = runNumberString.split('-')
if len(runs) != 2:
return S_ERROR('Could not determine run range from "%s", try "<Run 1> - <Run2>"' % (runNumberString))
try:
start = int(runs[0])
end = int(runs[1])
except Exception:
return S_ERROR('Invalid run range: %s' % runNumberString)
startRun = min(start, end)
endRun = max(start, end)
else:
try:
startRun = int(runNumberString)
endRun = startRun
except Exception:
return S_ERROR('Invalid run number: %s' % runNumberString)
query = self._bkQueryTemplate.copy()
query['StartRun'] = startRun
query['EndRun'] = endRun
query['ProcessingPass'] = bkPath[1]
query['EventType'] = bkPath[2]
query['FileType'] = bkPath[3]
if dqFlag:
check = self.__checkDQFlags(dqFlag)
if not check['OK']:
return check
dqFlag = check['Value']
query['DataQuality'] = dqFlag
result = self.bkQuery(query)
self.log.verbose(result)
return result
#############################################################################
def bkQueryProduction(self, bkPath, dqFlag='All'):
""" This function allows to create and perform a BK query given a supplied
BK path. The following BK path convention is expected:
/<ProductionID>/[<Processing Pass>/<Event Type>/]<File Type>
so an example for 2009 collisions data would be:
/5842/Real Data + RecoToDST-07/90000000/DST
Note that neither the processing pass nor the event type should be necessary. So either of them can be omitted
a data quality flag can also optionally be provided, the full list of these is available
via the getAllDQFlags() method.
Example Usage:
>>> dirac.bkQueryProduction('/5842/Real Data/RecoToDST-07/90000000/DST')
{'OK': True, 'Value': [<LFN1>,<LFN2>]}
@param bkPath: BK path as described above
@type bkPath: string
@param dqFlag: Optional Data Quality flag
@type dqFlag: string
@return: S_OK,S_ERROR
"""
if not isinstance(bkPath, str):
return S_ERROR('Expected string for bkPath')
# remove any double slashes, spaces must be preserved
# remove any empty components from leading and trailing slashes
bkPath = translateBKPath(bkPath, procPassID=1)
if len(bkPath) < 2:
return S_ERROR('Invalid bkPath: should at least contain /ProductionID/FileType')
query = self._bkQueryTemplate.copy()
try:
query['Production'] = int(bkPath[0])
except Exception:
return S_ERROR('Invalid production ID')
query['FileType'] = bkPath[-1]
if dqFlag:
check = self.__checkDQFlags(dqFlag)
if not check['OK']:
return check
dqFlag = check['Value']
query['DataQuality'] = dqFlag
for key, val in query.items():
if isinstance(val, basestring) and val.lower() == 'all':
query.pop(key)
result = self.bkQuery(query)
self.log.verbose(result)
return result
#############################################################################
def bkQueryPath(self, bkPath, dqFlag='All'):
""" This function allows to create and perform a BK query given a supplied
BK path. The following BK path convention is expected:
/<ConfigurationName>/<Configuration Version>/<Sim or Data Taking Condition>
/<Processing Pass>/<Event Type>/<File Type>
so an example for 2009 collsions data would be:
/LHCb/Collision09/Beam450GeV-VeloOpen-MagDown/Real Data + RecoToDST-07/90000000/DST
or for MC09 simulated data:
/MC/2010/Beam3500GeV-VeloClosed-MagDown-Nu1/2010-Sim01Reco01-withTruth/27163001/DST
a data quality flag can also optionally be provided, the full list of these is available
via the getAllDQFlags() method.
Example Usage:
>>> dirac.bkQueryPath('/MC/2010/Beam3500GeV-VeloClosed-MagDown-Nu1/Sim07/Reco06-withTruth/10012004/DST')
{'OK': True, 'Value': [<LFN1>,<LFN2>]}
@param bkPath: BK path as described above
@type bkPath: string
@param dqFlag: Optional Data Quality flag
@type dqFlag: string
@return: S_OK,S_ERROR
"""
if not isinstance(bkPath, str):
return S_ERROR('Expected string for bkPath')
# remove any double slashes, spaces must be preserved
# remove any empty components from leading and trailing slashes
bkPath = translateBKPath(bkPath, procPassID=3)
if not len(bkPath) == 6:
return S_ERROR('Expected 6 components to the BK path: '
'/<ConfigurationName>/<Configuration Version>/<Sim or Data Taking Condition>'
'/<Processing Pass>/<Event Type>/<File Type>')
query = self._bkQueryTemplate.copy()
query['ConfigName'] = bkPath[0]
query['ConfigVersion'] = bkPath[1]
query['ProcessingPass'] = bkPath[3]
query['EventType'] = bkPath[4]
query['FileType'] = bkPath[5]
if dqFlag:
check = self.__checkDQFlags(dqFlag)
if not check['OK']:
return check
dqFlag = check['Value']
query['DataQuality'] = dqFlag
# The problem here is that we don't know if it's a sim or data taking condition,
# assume that if configName=MC this is simulation
if bkPath[0].lower() == 'mc':
query['SimulationConditions'] = bkPath[2]
else:
query['DataTakingConditions'] = bkPath[2]
result = self.bkQuery(query)
self.log.verbose(result)
return result
#############################################################################
def bookkeepingQuery(self, SimulationConditions='All', DataTakingConditions='All',
ProcessingPass='******', FileType='All', EventType='All', ConfigName='All',
ConfigVersion='All', ProductionID=0, DataQuality='ALL'):
""" This function will create and perform a BK query using the supplied arguments
and return a list of LFNs.
Example Usage:
>>> dirac.bookkeepingQuery(ConfigName='LHCb',ConfigVersion='Collision09',
EventType='90000000',ProcessingPass='******',DataTakingConditions='Beam450GeV-VeloOpen-MagDown')
{'OK':True,'Value':<files>}
@param ConfigName: BK ConfigName
@type ConfigName: string
@param EventType: BK EventType
@type EventType: string
@param FileType: BK FileType
@type FileType: string
@param ProcessingPass: BK ProcessingPass
@type ProcessingPass: string
@param ProductionID: BK ProductionID
@type ProductionID: integer
@param DataQuality: BK DataQuality
@type DataQuality: string
@param ConfigVersion: BK ConfigVersion
@type ConfigVersion: string
@param DataTakingConditions: BK DataTakingConditions
@type DataTakingConditions: string
@param SimulationConditions: BK SimulationConditions
@type SimulationConditions: string
@return: S_OK,S_ERROR
"""
query = self._bkQueryTemplate.copy()
query['SimulationConditions'] = SimulationConditions
query['DataTakingConditions'] = DataTakingConditions
query['ProcessingPass'] = ProcessingPass
query['FileType'] = FileType
query['EventType'] = EventType
query['ConfigName'] = ConfigName
query['ConfigVersion'] = ConfigVersion
query['Production'] = ProductionID
query['DataQuality'] = DataQuality
return self.bkQuery(query)
#############################################################################
def bkQuery(self, bkQueryDict):
""" Developer function. Perform a query to the LHCb Bookkeeping to return
a list of LFN(s). This method takes a BK query dictionary.
Example Usage:
>>> print dirac.bkQuery(query)
{'OK':True,'Value':<files>}
@param bkQueryDict: BK query
@type bkQueryDict: dictionary (see bookkeepingQuery() for keys)
@return: S_OK,S_ERROR
"""
problematicFields = []
# Remove the Visible flag as anyway the method is for visible files ;-)
# bkQueryDict.setdefault( 'Visible', 'Yes' )
for name, value in bkQueryDict.items():
if name not in self._bkQueryTemplate:
problematicFields.append(name)
if problematicFields:
msg = 'The following fields are not valid for a BK query: %s\nValid fields include: %s' % \
(', '.join(problematicFields), ', '.join(self._bkQueryTemplate.keys()))
return S_ERROR(msg)
for name, value in bkQueryDict.items():
if name == "Production" or name == "EventType" or name == "StartRun" or name == "EndRun":
if value == 0:
del bkQueryDict[name]
else:
bkQueryDict[name] = str(value)
elif name == "FileType":
if value.lower() == "all":
bkQueryDict[name] = 'ALL'
else:
if str(value).lower() == "all":
del bkQueryDict[name]
if 'Production' in bkQueryDict or 'StartRun' in bkQueryDict or 'EndRun' in bkQueryDict:
self.log.verbose('Found a specific query so loosening some restrictions to prevent BK overloading')
else:
if 'SimulationConditions' not in bkQueryDict and 'DataTakingConditions' not in bkQueryDict:
return S_ERROR('A Simulation or DataTaking Condition must be specified for a BK query.')
if 'EventType' not in bkQueryDict and 'ConfigName' not in bkQueryDict and 'ConfigVersion' not in bkQueryDict:
return S_ERROR(
'The minimal set of BK fields for a query is: EventType, ConfigName and ConfigVersion'
' in addition to a Simulation or DataTaking Condition')
self.log.verbose('Final BK query dictionary is:')
for item in bkQueryDict.iteritems():
self.log.verbose('%s : %s' % item)
start = time.time()
result = self._bkClient.getVisibleFilesWithMetadata(bkQueryDict)
# result = bk.getFilesWithGivenDataSets(bkQueryDict)
rtime = time.time() - start
self.log.info('BK query time: %.2f sec' % rtime)
if not result['OK']:
return S_ERROR('BK query returned an error: "%s"' % (result['Message']))
if not result['Value']:
return self._errorReport('No BK files selected')
returnedFiles = len(result['Value'])
self.log.verbose('%s files selected from the BK' % (returnedFiles))
return result
#############################################################################
def __checkDQFlags(self, flags):
""" Internal function. Checks the provided flags against the list of
possible DQ flag statuses from the Bookkeeping.
"""
dqFlags = []
if isinstance(flags, list):
dqFlags = flags
else:
dqFlags = [flags]
bkFlags = self.getAllDQFlags()
if not bkFlags['OK']:
return bkFlags
final = []
for flag in dqFlags:
if flag.lower() == 'all':
final.append(flag.upper())
else:
flag = flag.upper()
if flag not in bkFlags['Value']:
msg = 'Specified DQ flag "%s" is not in allowed list: %s' % (flag, ', '.join(bkFlags['Value']))
self.log.error(msg)
return S_ERROR(msg)
else:
final.append(flag)
# when first coding this it was not possible to use a list ;)
if len(final) == 1:
final = final[0]
return S_OK(final)
#############################################################################
def getAllDQFlags(self, printOutput=False):
""" Helper function. Returns the list of possible DQ flag statuses
from the Bookkeeping.
Example Usage:
>>> print dirac.getAllDQFlags()
{'OK':True,'Value':<flags>}
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
result = self._bkClient.getAvailableDataQuality()
if not result['OK']:
self.log.error('Could not obtain possible DQ flags from BK with result:\n%s' % (result))
return result
if printOutput:
flags = result['Value']
self.log.info('Possible DQ flags from BK are: %s' % (', '.join(flags)))
return result
#############################################################################
def getDataByRun(self, lfns, printOutput=False):
"""Sort the supplied lfn list by run. An S_OK object will be returned
containing a dictionary of runs and the corresponding list of LFN(s)
associated with them.
Example usage:
>>> print dirac.getDataByRun(lfns)
{'OK': True, 'Value': {<RUN>:['<LFN>','<LFN>',...], <RUN>:['<LFN>',..]}}
@param lfns: Logical File Name(s)
@type lfns: list
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
if isinstance(lfns, str):
lfns = [lfns.replace('LFN:', '')]
elif isinstance(lfns, list):
try:
lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns]
except ValueError as x:
return self._errorReport(str(x), 'Expected strings for LFNs')
else:
return self._errorReport('Expected single string or list of strings for LFN(s)')
runDict = {}
start = time.time()
result = self._bkClient.getFileMetadata(lfns)
self.log.verbose("Obtained BK file metadata in %.2f seconds" % (time.time() - start))
if not result['OK']:
self.log.error('Failed to get bookkeeping metadata with result "%s"' % (result['Message']))
return result
for lfn, metadata in result['Value']['Successful'].items():
if 'RunNumber' in metadata:
runNumber = metadata['RunNumber']
runDict.setdefault(runNumber, []).append(lfn)
else:
self.log.warn('Could not find run number from BK for %s' % (lfn))
if printOutput:
self.log.notice(self.pPrint.pformat(runDict))
return S_OK(runDict)
#############################################################################
def bkMetadata(self, lfns, printOutput=False):
"""Return metadata for the supplied lfn list. An S_OK object will be returned
containing a dictionary of LFN(s) and the corresponding metadata associated
with them.
Example usage:
>>> print dirac.bkMetadata(lfns)
{'OK': True, 'Value': {<LFN>:{'<Name>':'<Value>',...},...}}
@param lfns: Logical File Name(s)
@type lfns: list
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
if isinstance(lfns, str):
lfns = [lfns.replace('LFN:', '')]
elif isinstance(lfns, list):
try:
lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns]
except ValueError as x:
return self._errorReport(str(x), 'Expected strings for LFNs')
else:
return self._errorReport('Expected single string or list of strings for LFN(s)')
start = time.time()
result = self._bkClient.getFileMetadata(lfns)
self.log.verbose("Obtained BK file metadata in %.2f seconds" % (time.time() - start))
if not result['OK']:
self.log.error('Failed to get bookkeeping metadata with result "%s"' % (result['Message']))
return result
if printOutput:
self.log.notice(self.pPrint.pformat(result['Value']))
return result
#############################################################################
def lhcbProxyInit(self, *args): # pylint: disable=no-self-use
""" just calling the dirac-proxy-init script
"""
os.system("dirac-proxy-init -o LogLevel=NOTICE -t --rfc %s" % "' '".join(args))
#############################################################################
def lhcbProxyInfo(self, *args): # pylint: disable=no-self-use
""" just calling the dirac-proxy-info script
"""
os.system("dirac-proxy-info -o LogLevel=NOTICE %s" % "' '".join(args))
#############################################################################
def gridWeather(self, printOutput=False):
"""This method gives a snapshot of the current Grid weather from the perspective
of the DIRAC site and SE masks. Tier-1 sites are returned with more detailed
information.
Example usage:
>>> print dirac.gridWeather()
{'OK': True, 'Value': {{'Sites':<siteInfo>,'SEs':<seInfo>,'Tier-1s':<tierInfo>}}
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
lcgSites = gConfig.getSections('/Resources/Sites/LCG')
if not lcgSites['OK']:
return lcgSites
for lcgSite in lcgSites['Value']:
tier = gConfig.getValue('/Resources/Sites/LCG/%s/MoUTierLevel' % lcgSite, 2)
if tier in (0, 1):
self.tier1s.append(lcgSite)
siteInfo = self.checkSites()
if not siteInfo['OK']:
return siteInfo
siteInfo = siteInfo['Value']
seInfo = self.checkSEs()
if not seInfo['OK']:
return seInfo
seInfo = seInfo['Value']
tierSEs = {}
for site in self.tier1s:
tierSEs[site] = getSEsForSite(site)['Value']
tierInfo = {}
for site, seList in tierSEs.items():
tierInfo[site] = {}
for se in seList:
if se in seInfo:
tierSEInfo = seInfo[se]
tierInfo[site][se] = tierSEInfo
if site in siteInfo['AllowedSites']:
tierInfo[site]['MaskStatus'] = 'Allowed'
else:
tierInfo[site]['MaskStatus'] = 'Banned'
if printOutput:
self.log.notice('========> Tier-1 status in DIRAC site and SE masks')
for site in sorted(self.tier1s):
self.log.notice('\n====> %s is %s in site mask\n' % (site, tierInfo[site]['MaskStatus']))
self.log.notice('%s %s %s' % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15)))
for se in sorted(tierSEs[site]):
if se in tierInfo[site]:
self.log.notice('%s %s %s' % (se.ljust(25),
tierInfo[site][se]['ReadStatus'].rjust(15),
tierInfo[site][se]['WriteStatus'].rjust(15))
)
self.log.notice('\n========> Tier-2 status in DIRAC site mask\n')
allowedSites = siteInfo['AllowedSites']
bannedSites = siteInfo['BannedSites']
for site in self.tier1s:
if site in allowedSites:
allowedSites.remove(site)
if site in bannedSites:
bannedSites.remove(site)
self.log.notice(' %s sites are in the site mask, %s are banned.\n' % (len(allowedSites), len(bannedSites)))
summary = {'Sites': siteInfo, 'SEs': seInfo, 'Tier-1s': tierInfo}
return S_OK(summary)
#############################################################################
def checkSites(self, printOutput=False): # pylint: disable=no-self-use
"""Return the list of sites in the DIRAC site mask and those which are banned.
Example usage:
>>> print dirac.checkSites()
{'OK': True, 'Value': {'AllowedSites':['<Site>',...],'BannedSites':[]}
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
res = getSites()
if not res['OK']:
self.log.error('Could not get list of sites from CS', res['Message'])
return res
totalList = res['Value']
res = DiracAdmin().getSiteMask()
if not res['OK']:
return res
sites = res['Value']
bannedSites = []
for site in totalList:
if site not in sites:
bannedSites.append(site)
if printOutput:
self.log.notice('\n========> Allowed Sites\n')
self.log.notice('\n'.join(sites))
self.log.notice('\n========> Banned Sites\n')
self.log.notice('\n'.join(bannedSites))
self.log.notice('\nThere is a total of %s allowed sites and %s banned sites in the system.' % (len(sites),
len(bannedSites)))
return S_OK({'AllowedSites': sites, 'BannedSites': bannedSites})
#############################################################################
def checkSEs(self, printOutput=False): # pylint: disable=no-self-use
"""Check the status of read and write operations in the DIRAC SE mask.
Example usage:
>>> print dirac.checkSEs()
{'OK': True, 'Value': {<LFN>:{'<Name>':'<Value>',...},...}}
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
res = gConfig.getSections('/Resources/StorageElements', True)
if not res['OK']:
self.log.error('Failed to get storage element information', res['Message'])
return res
if printOutput:
self.log.notice('%s %s %s' % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15)))
seList = sorted(res['Value'])
result = {}
rss = ResourceStatus()
for se in seList:
res = rss.getElementStatus(se, 'StorageElement')
if not res['OK']:
self.log.error("Failed to get StorageElement status for %s" % se)
else:
readState = res['Value'].get('ReadAccess', 'Active')
writeState = res['Value'].get('WriteAccess', 'Active')
result[se] = {'ReadStatus': readState, 'WriteStatus': writeState}
if printOutput:
self.log.notice('%s %s %s' % (se.ljust(25), readState.rjust(15), writeState.rjust(15)))
return S_OK(result)
def splitInputDataBySize(self, lfns, maxSizePerJob=20, printOutput=False):
"""Split the supplied lfn list by the replicas present at the possible
destination sites, based on a maximum size.
An S_OK object will be returned containing a list of
lists in order to create the jobs.
Example usage:
>>> d.splitInputDataBySize(lfns,10)
{'OK': True, 'Value': [['<LFN>'], ['<LFN>']]}
@param lfns: Logical File Name(s) to split
@type lfns: list
@param maxSizePerJob: Maximum size (in GB) per bunch
@type maxSizePerJob: integer
@param printOutput: Optional flag to print result
@type printOutput: boolean
@return: S_OK,S_ERROR
"""
sitesForSE = {}
if isinstance(lfns, str):
lfns = [lfns.replace('LFN:', '')]
elif isinstance(lfns, list):
try:
lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns]
except TypeError as x:
return self._errorReport(str(x), 'Expected strings for LFNs')
else:
return self._errorReport('Expected single string or list of strings for LFN(s)')
if not isinstance(maxSizePerJob, int):
try:
maxSizePerJob = int(maxSizePerJob)
except ValueError as x:
return self._errorReport(str(x), 'Expected integer for maxSizePerJob')
maxSizePerJob *= 1000 * 1000 * 1000
replicaDict = self.getReplicas(lfns)
if not replicaDict['OK']:
return replicaDict
replicas = replicaDict['Value']['Successful']
if not replicas:
return self._errorReport(replicaDict['Value']['Failed'].items()[0],
'Failed to get replica information')
siteLfns = {}
for lfn, reps in replicas.items():
possibleSites = set(site
for se in reps
for site in sitesForSE.setdefault(se, getSitesForSE(se).get('Value', [])))
siteLfns.setdefault(','.join(sorted(possibleSites)), []).append(lfn)
if '' in siteLfns:
# Some files don't have active replicas
return self._errorReport('No active replica found for', str(siteLfns['']))
# Get size of files
metadataDict = self.getLfnMetadata(lfns, printOutput)
if not metadataDict['OK']:
return metadataDict
fileSizes = dict((lfn, metadataDict['Value']['Successful'].get(lfn, {}).get('Size', maxSizePerJob))
for lfn in lfns)
lfnGroups = []
# maxSize is in GB
for files in siteLfns.values():
# Now get bunches of files,
# Sort in decreasing size
files.sort(cmp=(lambda f1, f2: fileSizes[f2] - fileSizes[f1]))
while files:
# print [( lfn, fileSizes[lfn] ) for lfn in files]
group = []
sizeTot = 0
for lfn in list(files):
size = fileSizes[lfn]
if size >= maxSizePerJob:
lfnGroups.append([lfn])
elif sizeTot + size < maxSizePerJob:
sizeTot += size
group.append(lfn)
files.remove(lfn)
if group:
lfnGroups.append(group)
if printOutput:
self.log.notice(self.pPrint.pformat(lfnGroups))
return S_OK(lfnGroups)
#############################################################################
def getAccessURL(self, lfn, storageElement, protocol=None, printOutput=False):
"""Allows to retrieve an access URL for an LFN replica given a valid DIRAC SE
name. Contacts the file catalog and contacts the site SRM endpoint behind
the scenes.
Example Usage:
>>> print dirac.getAccessURL('/lhcb/data/CCRC08/DST/00000151/0000/00000151_00004848_2.dst','CERN-RAW')
{'OK': True, 'Value': {'Successful': {'srm://...': {'SRM2': 'rfio://...'}}, 'Failed': {}}}
:param lfn: Logical File Name (LFN)
:type lfn: str or python:list
:param storageElement: DIRAC SE name e.g. CERN-RAW
:type storageElement: string
:param printOutput: Optional flag to print result
:type printOutput: boolean
:returns: S_OK,S_ERROR
"""
ret = self._checkFileArgument(lfn, 'LFN')
if not ret['OK']:
return ret
lfn = ret['Value']
if isinstance(lfn, basestring):
lfn = [lfn]
results = getAccessURL(lfn, storageElement, protocol=protocol)
if printOutput:
printDMResult(results, empty="File not at SE", script="dirac-dms-lfn-accessURL")
return results
#############################################################################
def _getLocalInputData(self, parameters):
""" LHCb extension of DIRAC API's _getLocalInputData. Only used for handling ancestors.
"""
inputData = parameters.get('InputData')
if inputData:
self.log.debug("DiracLHCb._getLocalInputData. InputData: %s" % inputData)
if isinstance(inputData, basestring):
inputData = inputData.split(';')
inputData = [lfn.strip('LFN:') for lfn in inputData]
ancestorsDepth = int(parameters.get('AncestorDepth', 0))
if ancestorsDepth:
self.log.debug("DiracLHCb._getLocalInputData. ancestorsDepth: %d" % ancestorsDepth)
res = self._bkClient.getFileAncestors(inputData, ancestorsDepth)
if not res['OK']:
self.log.error("Can't get ancestors", res['Message'])
return res
ancestorsLFNs = []
for ancestorsLFN in res['Value']['Successful'].itervalues():
ancestorsLFNs += [i['FileName'] for i in ancestorsLFN]
self.log.info("DiracLHCb._getLocalInputData: adding %d ancestors" % len(ancestorsLFNs))
self.log.verbose("%s", ', '.join(ancestorsLFNs))
inputData += ancestorsLFNs
return S_OK(inputData)
class ProcessingProgress( object ):
def __init__( self, cacheFile = None ):
if not cacheFile:
self.prodStatFile = os.path.join( os.environ['HOME'], ".dirac/work", "dirac-production-stats.pkl" )
else:
self.prodStatFile = cacheFile
self.cacheVersion = '0.0'
self.clearCache = []
self.cachedInfo = {}
# Recuperate the previous cached information
self.readCache()
self.bk = BookkeepingClient()
self.transClient = TransformationClient()
def setClearCache( self, clearCache ):
self.clearCache = clearCache
def __getProdBkDict( self, prod ):
res = self.transClient.getBookkeepingQuery( prod )
if not res['OK']:
gLogger.error( "Couldn't get BK query on production %d" % prod )
return {}
prodBKDict = res['Value']
return prodBKDict
def getFullStats( self, bkQuery, printResult = False ):
processingPass = bkQuery.getProcessingPass()
if printResult:
gLogger.info( "\nStatistics for processing %s, condition %s\n" % ( processingPass, bkQuery.getConditions() ) )
prodStats = []
processingPass = processingPass.split( '/' )
if len( processingPass ) != 4 or processingPass[1] != "Real Data":
gLogger.error( "Processing pass should be /Real Data/<Reco>/<Stripping>" )
return []
# Get production numbers for the Reco
recoBKQuery = BKQuery( bkQuery )
recoBKQuery.setProcessingPass( '/'.join( processingPass[0:3] ) )
recoList = recoBKQuery.getBKProductions( visible = False )
recoRunRanges = {}
recoDQFlags = []
for prod in recoList:
prodBKDict = self.__getProdBkDict( prod )
if prodBKDict:
recoRunRanges[prod] = [prodBKDict.get( "StartRun", 0 ), prodBKDict.get( "EndRun", sys.maxint )]
dqFlags = prodBKDict.get( "DataQualityFlag", ['UNCHECKED', 'EXPRESS_OK', 'OK'] )
if isinstance( dqFlags, basestring ):
dqFlags = dqFlags.split( ',' )
recoDQFlags += [fl for fl in dqFlags if fl not in recoDQFlags]
else:
recoRunRanges[prod] = [0, 0]
# Sort productions by runs
try:
recoList.sort( cmp = ( lambda p1, p2: int( recoRunRanges[p1][0] - recoRunRanges[p2][1] ) ) )
except:
print "Exception in sorting productions:"
for p in recoList:
print p, recoRunRanges[p]
gLogger.verbose( "Reconstruction productions found (%d): %s" % ( len( recoList ), str( sorted( recoList ) ) ) )
gLogger.verbose( "Reconstruction DQ flags: %s" % str( recoDQFlags ) )
# Get productions for merging
mergeList = []
mergeStripProds = {}
# Get stripping productions as parents of merging productions
stripList = []
for prod in bkQuery.getBKProductions( visible = False ):
prodBKDict = self.__getProdBkDict( prod )
gLogger.verbose( "BK query for production %s: %s" % ( prod, str( prodBKDict ) ) )
mergedTypes = prodBKDict.get( 'FileType' )
if type( mergedTypes ) != type( [] ):
mergedTypes = [mergedTypes]
if [ft for ft in bkQuery.getFileTypeList() if ft in mergedTypes] and 'ProductionID' in prodBKDict:
mergeList.append( prod )
prods = prodBKDict['ProductionID']
if type( prods ) != type( [] ):
prods = [prods]
stripList += prods
mergeStripProds[prod] = [str( p ) for p in prods]
else:
_msgTuple = ( str( bkQuery.getFileTypeList() ), prod, str( prodBKDict ) )
gLogger.verbose( "Could not find production or filetype %s in BKquery of production %d (%s)" % _msgTuple )
mergeList.sort( cmp = ( lambda p1, p2: int( mergeStripProds[p1][0] ) - int( mergeStripProds[p2][0] ) ) )
gLogger.verbose( "Merging productions found: %s" % str( mergeList ) )
# get list of stripping productions (from merging)
stripRunRanges = {}
for prod in stripList:
prodBKDict = self.__getProdBkDict( prod )
if prodBKDict:
stripRunRanges[prod] = [prodBKDict.get( "StartRun", 0 ), prodBKDict.get( "EndRun", sys.maxint )]
else:
stripRunRanges[prod] = [0, 0]
# Sort productions by runs
try:
stripList.sort( cmp = ( lambda p1, p2: int( stripRunRanges[p1][0] - stripRunRanges[p2][1] ) ) )
except:
print "Error when sorting stripping productions:"
for prodStrip in stripList:
print prodStrip, stripRunRanges[prodStrip]
gLogger.verbose( "Stripping productions found (%d): %s" % ( len( stripList ), str( sorted( stripList ) ) ) )
# Get all runs corresponding to the run range used by the Reco productions
rawBKQuery = BKQuery( bkQuery )
rawBKQuery.setProcessingPass( '/Real Data' )
rawBKQuery.setFileType( "RAW" )
# get the list of runs (-prodNum)
fullRunList = rawBKQuery.getBKRuns()
gLogger.verbose( "Initial list of runs: %s" % str( fullRunList ) )
recoRunList = []
openProd = False
for prod in [p for p in recoList]:
# Forget fully openProd productions
# Don't consider productions without a BK query (these are individual files)
if recoRunRanges[prod][1] == sys.maxint and recoRunRanges[prod][0] != -sys.maxint:
openProd = True
# Try and find if that open production overlaps with a closed one, in which case, remove it
# Do nothing for derived productions
for p in [p for p in recoList if p != prod and recoRunRanges[prod] != recoRunRanges[p]]:
if recoRunRanges[prod][0] < recoRunRanges[p][1] and recoRunRanges[p][1] != sys.maxint:
openProd = False
gLogger.verbose( "Production %s was removed as redundant..." % str( prod ) )
recoList.remove( prod )
break
if not openProd: continue
recoRunList += [run for run in fullRunList if run not in recoRunList and run >= recoRunRanges[prod][0] and run <= recoRunRanges[prod][1]]
gLogger.verbose( "List of runs matching Reco (%d): %s" % ( len( recoRunList ), str( sorted( recoRunList ) ) ) )
restrictToStripping = True
if restrictToStripping and not openProd and stripList:
runList = []
for prod in stripList:
runList += [run for run in recoRunList if run not in runList and run >= stripRunRanges[prod][0] and run <= stripRunRanges[prod][1]]
else:
runList = recoRunList
gLogger.verbose( "Final list of runs matching Reco and Stripping (%d): %s" % ( len( runList ), str( sorted( runList ) ) ) )
# Now get statistics from the runs
info, runInfo = self._getStatsFromRuns( int( bkQuery.getEventTypeList()[0] ), runList, recoDQFlags )
rawInfo = StatInfo( processingPass[1], info )
prodStats.append( rawInfo )
if printResult:
gLogger.info( "%s - All runs in Reco productions" % processingPass[1] )
for fileInfo in runInfo:
if runInfo[fileInfo]:
gLogger.info( "%s runs (%d): %s" % ( fileInfo, len( runInfo[fileInfo] ), str( runInfo[fileInfo] ) ) )
summStr = "%s files, " % rawInfo.getItemAsString( 'Files' )
summStr += "%s events in " % rawInfo.getItemAsString( 'Events' )
summStr += "%s runs, luminosity (pb-1):All=%s, Bad=%s, OK=%s" % ( rawInfo.getItemAsString( 'Runs' ),
rawInfo.getItemAsString( 'Lumi' ),
rawInfo.getItemAsString( 'BadLumi' ),
rawInfo.getItemAsString( 'OKLumi' ) )
gLogger.info( summStr )
# Create the info for the 3 sets of productions
prodSets = []
fileType = bkQuery.getFileTypeList()[0]
prodSets.append( {'Name': processingPass[2], 'FileType': ['SDST', 'FULL.DST'], 'List':recoList,
'RunRange':recoRunRanges, 'MotherProds':None, 'AllReplicas':False } )
prodSets.append( {'Name': processingPass[3], 'FileType': fileType, 'List':stripList,
'RunRange':stripRunRanges, 'MotherProds':None, 'AllReplicas':True, 'StatForOK':False } )
prodSets.append( {'Name': "Merging (%s)" % fileType.split( '.' )[0], 'FileType': fileType, 'List':mergeList,
'RunRange':None, 'MotherProds':mergeStripProds, 'AllReplicas':False } )
prevInfo = rawInfo
for prodSet in prodSets:
info = StatInfo( prodSet['Name'], self._getProdInfo( prodSet, runList, printResult = printResult ) )
info.setRawInfo( rawInfo )
info.setPrevInfo( prevInfo )
prevInfo = info
prodStats.append( info )
self.saveCache()
return prodStats
@staticmethod
def __sumProdInfo( info, totInfo ):
for inf in info:
for flag in info[inf]:
if inf == 'Runs':
totInfo[inf][flag] = totInfo.setdefault( inf, {} ).setdefault( flag, [] ) + info[inf][flag]
else:
totInfo[inf][flag] = totInfo.setdefault( inf, {} ).setdefault( flag, 0 ) + info[inf][flag]
return totInfo
def _getProdInfo( self, prodSet, runList, printResult = False ):
totInfo = {}
if printResult:
gLogger.info( "" )
for prod in prodSet['List']:
info, runInfo = self._getStatsFromBK( prod, prodSet['FileType'], runList, prodSet['AllReplicas'] )
if info['Files'][''] == 0:
continue
if not prodSet.get( 'StatForOK', True ):
for item in info:
for fl in info[item]:
if fl == 'OK':
info[item][fl] = 0 if not item == 'Runs' else []
runRange = prodSet['RunRange']
if runRange and prod in runRange and runRange[prod][0] == 0 and runRange[prod][1] == 0:
for flag in info['Runs']:
info['Runs'][flag] = []
totInfo = self.__sumProdInfo( info, totInfo )
if printResult:
summStr = "%s production %d -" % ( prodSet['Name'], prod )
if runRange and prod in runRange:
firstRun = runRange[prod][0]
lastRun = runRange[prod][1]
if firstRun:
summStr += " From run %d" % int( firstRun )
if lastRun and lastRun != sys.maxint:
summStr += " Up to run %d" % int( lastRun )
if firstRun == 0 and lastRun == 0:
summStr += "No run range specified"
motherProds = prodSet['MotherProds']
if motherProds and prod in motherProds:
summStr += " from productions %s" % motherProds[prod]
gLogger.info( summStr )
for inf in runInfo:
if runInfo[inf]:
gLogger.info( "%s runs (%d): %s" % ( inf, len( runInfo[inf] ), str( runInfo[inf] ) ) )
summStr = "%d files, " % info['Files']['']
if info['Events']:
summStr += "%d events in " % info['Events']['']
_msgTuple = ( len( info['Runs'][''] ), info['Lumi'][''], info['Lumi']['Bad'], info['Lumi']['OK'] )
summStr += "%d runs, luminosity (pb-1): All=%.3f, Bad=%.3f, OK=%.3f" % _msgTuple
gLogger.info( summStr )
for flag in totInfo.get( 'Runs', [] ):
totInfo['Runs'][flag] = len( totInfo['Runs'][flag] )
return totInfo
@staticmethod
def outputResults( conditions, processingPass, prodStats ):
outputString = ""
_msgTuple = ( conditions, ",", processingPass, "on", time.ctime( time.time() ) )
outputString += "\nProduction progress for %s %s %s %s %s\n" % _msgTuple
if len( prodStats ) < 4:
outputString += "No statistics found for this BK query"
return outputString
for i in xrange( 4 ):
info = prodStats[i]
if not info:
continue
name = info.getName()
outputString += "\nSummary for %s\n" % name
outputString += "%d files, " % info.getItem( 'Files' )
if info.getItem( 'Events' ):
outputString += "%d events in " % info.getItem( 'Events' )
_msgTuple = ( info.getItem( 'Runs' ), info.getItem( 'Lumi' ), info.getItem( 'BadLumi' ), info.getItem( 'OKLumi' ) )
outputString += "%d runs, luminosity (pb-1): All=%.3f, Bad=%.3f, OK=%.3f\n" % _msgTuple
prevStats = prodStats[:i]
prevStats.reverse()
for prevInfo in prevStats:
name = prevInfo.getName()
if prevInfo.getItem( 'Runs' ) == 0:
outputString += "From %s : - No runs...\n" % name
else:
outputString += "From %s : %.1f%% files, " % ( name, 100.*info.getItem( 'Files' ) / prevInfo.getItem( 'Files' ) )
if info.getItem( 'Events' ) and prevInfo.getItem( 'Events' ):
outputString += "%.1f%% events\n" % ( 100.*info.getItem( 'Events' ) / prevInfo.getItem( 'Events' ) )
outputString += "%.1f%% runs, %.1f%% luminosity\n" \
% ( 100. * info.getItem( 'Runs' ) / prevInfo.getItem( 'Runs' ),
100. * info.getItem( 'Lumi' ) / prevInfo.getItem( 'Lumi' ) )
return outputString
def __getRunsDQFlag( self, runList, evtType ):
res = self.bk.getRunFilesDataQuality( runList )
runFlags = {}
if res['OK']:
dqFlags = res['Value']
for dq in dqFlags:
if dq[2] == evtType:
runFlags.setdefault( dq[0], [] ).append( dq[1] )
runDQFlags = {}
flags = ( 'BAD', 'OK', 'EXPRESS_OK', 'UNCHECKED' )
for run in runFlags:
for fl in flags:
if fl in runFlags[run]:
runDQFlags[run] = fl
break
return runDQFlags
def _getStatsFromRuns( self, evtType, runList, recoDQFlags ):
info = dict.fromkeys( ( 'Events', 'Runs', 'Files', 'Lumi' ), {} )
for inf in info:
info[inf] = dict.fromkeys( ( 'Bad', 'OK', '' ), 0 )
now = datetime.datetime.utcnow()
# Set to True to renew the cache
clearCache = 'RAW' in self.clearCache
newRuns = [ run for run in runList if clearCache
or run not in self.cachedInfo
or 'DQFlag' not in self.cachedInfo[run]
or ( now - self.cachedInfo[run]['Time'] ) < datetime.timedelta( days = 2 ) ]
if newRuns:
runFlags = self.__getRunsDQFlag( newRuns, evtType )
else:
runFlags = {}
runsByDQFlag = {}
runInfo = {}
for run in runList:
cached = self.cachedInfo.get( run, {} )
cachedTime = cached.get( 'Time', None )
if run not in newRuns:
cachedFiles = cached.get( 'Files', 0 )
cachedEvents = cached.get( 'EventStat', 0 )
cachedLumi = cached.get( 'Luminosity', 0 )
dqFlag = cached.get( 'DQFlag', None )
else:
res = self.bk.getRunInformations( run )
if res['OK']:
val = res['Value']
ind = val['Stream'].index( 90000000 )
cachedFiles = val['Number of file'][ind]
cachedEvents = val['Number of events'][ind]
cachedLumi = val['luminosity'][ind]
cachedTime = val['RunStart']
else:
gLogger.error( "Unable to get run information for run %s" % str( run ) )
continue
dqFlag = runFlags[run]
self.cachedInfo[run] = { 'Time':cachedTime, 'Files':cachedFiles, 'EventStat': cachedEvents,
'Luminosity': cachedLumi, 'DQFlag':dqFlag }
runsByDQFlag[dqFlag] = runsByDQFlag.setdefault( dqFlag, 0 ) + 1
if dqFlag == "BAD":
runInfo.setdefault( 'BAD', [] ).append( run )
elif dqFlag not in recoDQFlags and dqFlag != 'OK' :
runInfo.setdefault( 'Untagged', [] ).append( run )
# Now count...
flags = []
if dqFlag != 'BAD':
flags.append( '' )
# OK in recoDQFlags means we take everything that is not BAD (reprocessing or new convention)
if dqFlag in recoDQFlags or dqFlag == 'OK':
flags.append( 'OK' )
else:
flags.append( 'Bad' )
for flag in flags:
info['Runs'][flag] += 1
info['Files'][flag] += cachedFiles
info['Events'][flag] += cachedEvents
info['Lumi'][flag] += cachedLumi
# Set lumi in pb-1
for flag in info['Lumi']:
info['Lumi'][flag] /= 1000000.
gLogger.info( "Runs per flag:" )
for key in runsByDQFlag:
gLogger.info( "%s : %d" % ( key, runsByDQFlag[key] ) )
for flag in runInfo:
runInfo[flag].sort()
return info, runInfo
def __getLfnsMetadata( self, lfns ):
lfnDict = {}
if len( lfns ):
gLogger.verbose( "Getting metadata for %d files" % len( lfns ) )
for lfnChunk in breakListIntoChunks( lfns, 1000 ):
while True:
res = self.bk.getFileMetadata( lfnChunk )
if not res['OK']:
gLogger.error( "Error getting files metadata, retrying...", res['Message'] )
else:
break
metadata = res['Value']['Successful']
for lfn in lfnChunk:
lfnDict[lfn] = {}
for meta in ( 'EventStat', 'Luminosity', 'DQFlag', 'RunNumber' ):
lfnDict[lfn][meta] = metadata[lfn][meta]
return lfnDict
def _getStatsFromBK( self, prod, fileType, runList, allReplicas ):
bkQueryDict = { "ProductionID": prod, "FileType": fileType }
bkStr = str( bkQueryDict )
bkQuery = BKQuery( bkQueryDict, visible = not allReplicas )
if allReplicas:
bkQuery.setOption( 'ReplicaFlag', "All" )
cached = self.cachedInfo.get( bkStr, {} )
cachedTime = cached.get( 'Time', None )
cachedLfns = cached.get( 'Lfns', {} )
if isinstance( fileType, basestring ):
fileType = [fileType]
if set( fileType ).intersection( set( self.clearCache ) ):
cachedTime = datetime.datetime.utcnow() - datetime.timedelta( days = 8 )
cachedTime = None
cachedLfns = {}
gLogger.verbose( "Cleared cache for production %s, file type %s" % ( str( prod ), fileType ) )
# Update if needed the cached information on LFNs
if cachedLfns:
lfns = [lfn for lfn in cachedLfns if cachedLfns[lfn].get( 'DQFlag' ) not in ( 'OK', 'BAD' )]
for lfnChunk in breakListIntoChunks( lfns, 1000 ):
# get the DQFlag of files that are not yet OK
while True:
res = self.bk.getFileMetadata( lfnChunk )
if not res['OK']:
gLogger.error( "Error getting files metadata for cached files, bkQuery %s: %s" % ( bkStr, res['Message'] ) )
else:
metadata = res['Value']['Successful']
for lfn in lfnChunk:
cachedLfns[lfn]['DQFlag'] = metadata[lfn]['DQFlag']
break
# Now get the new files since last time...
if cachedTime:
bkQuery.setOption( 'StartDate', cachedTime.strftime( '%Y-%m-%d %H:%M:%S' ) )
gLogger.verbose( "Getting files for BKQuery %s" % str( bkQuery ) )
cachedTime = datetime.datetime.utcnow()
lfns = [lfn for lfn in bkQuery.getLFNs( printOutput = False ) if lfn not in cachedLfns]
gLogger.verbose( "Returned %d files" % len( lfns ) )
cachedLfns.update( self.__getLfnsMetadata( lfns ) )
self.cachedInfo[bkStr] = { 'Time':cachedTime, 'Lfns':cachedLfns }
# Now sum up all information for the files
info = dict.fromkeys( ( 'Events', 'Runs', 'Files', 'Lumi' ), {} )
for inf in info:
if inf == 'Runs':
for flag in ( 'Bad', 'OK', '' ):
info[inf][flag] = []
else:
info[inf] = dict.fromkeys( ( 'Bad', 'OK', '' ), 0 )
for lfn in cachedLfns:
lfnInfo = cachedLfns[lfn]
run = lfnInfo['RunNumber']
if run in runList and run in self.cachedInfo and self.cachedInfo[run]['DQFlag'] != 'BAD':
dqFlag = cachedLfns[lfn]['DQFlag']
flags = []
if dqFlag != 'BAD':
flags.append( '' )
else:
flags.append( 'Bad' )
if dqFlag == 'OK':
flags.append( 'OK' )
for flag in flags:
if run not in info['Runs'][flag]:
info['Runs'][flag].append( run )
info['Files'][flag] += 1
info['Events'][flag] += lfnInfo['EventStat']
info['Lumi'][flag] += lfnInfo['Luminosity']
runInfo = {}
if 'BAD' in info['Runs']:
runInfo['BAD'] = info['Runs']['BAD']
runInfo['BAD'].sort()
else:
runInfo['BAD'] = []
if '' in info['Runs'] and 'OK' in info['Runs']:
runInfo['Untagged'] = [run for run in info['Runs'][''] if run not in info['Runs']['OK']]
runInfo['Untagged'].sort()
else:
runInfo['Untagged'] = []
# for f in info['Runs']:
# info['Runs'][f] = len( info['Runs'][f] )
for flag in info['Lumi']:
info['Lumi'][flag] /= 1000000.
return info, runInfo
def getPreviousStats( self, processingPass ):
prevStats = self.cachedInfo.get( 'ProdStats', {} ).get( processingPass )
if prevStats:
try:
_name = prevStats['ProdStats'][0][0].getName()
except:
prevStats = None
return prevStats
def setPreviousStats( self, processingPass, prevStats ):
self.cachedInfo.setdefault( 'ProdStats', {} )[processingPass] = prevStats
self.saveCache()
def readCache( self ):
if not os.path.exists( self.prodStatFile ):
gLogger.info( "Created cached file %s" % self.prodStatFile )
self.cachedInfo = {}
self.saveCache()
return
fileRead = False
while not fileRead:
try:
with FileLock( self.prodStatFile ):
lFile = open( self.prodStatFile, 'r' )
cachedVersion = pickle.load( lFile )
startTime = time.time()
if cachedVersion == self.cacheVersion:
self.cachedInfo = pickle.load( lFile )
_msgTuple = ( self.prodStatFile, time.time() - startTime )
gLogger.info( "Loaded cached information from %s in %.3f seconds" % _msgTuple )
else:
_msgTuple = ( cachedVersion, self.cacheVersion )
gLogger.info( "Incompatible versions of cache, reset information (%s, expect %s)" % _msgTuple )
self.cachedInfo = {}
lFile.close()
fileRead = True
except FileLockException, error:
gLogger.error( "Lock exception: %s while reading pickle file %s" % ( error, self.prodStatFile ) )
except:
