def __call__(self):
reportData = self.collectReportData()
if not reportData:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Status information is unavailable, will not proceed with the report."
msg += " Try again a few minutes later if the task has just been submitted."
self.logger.info(msg)
return None
returndict = {}
if self.options.recovery == 'notPublished' and not reportData[
'publication']:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " The option --recovery=%s has been specified" % (
self.options.recovery)
msg += " (which instructs to determine the not processed lumis based on published datasets),"
msg += " but publication has been disabled in the CRAB configuration."
raise ConfigurationException(msg)
onlyDBSSummary = False
if not reportData['lumisToProcess'] or not reportData['runsAndLumis']:
msg = "%sError%s:" % (colors.RED, colors.NORMAL)
msg += " Cannot get all the needed information for the report."
msg += " Notice, if your task has been submitted more than 30 days ago, then everything has been cleaned."
self.logger.info(msg)
if not reportData['publication']:
return returndict
onlyDBSSummary = True
def _getNumFiles(jobs, fileType):
files = set()
for _, reports in jobs.iteritems():
for rep in reports:
if rep['type'] == fileType:
# the split is done to remove the jobnumber at the end of the input file lfn
files.add('_'.join(rep['lfn'].split('_')[:-1]))
return len(files)
def _getNumEvents(jobs, fileType):
numEvents = 0
for _, reports in jobs.iteritems():
for rep in reports:
if rep['type'] == fileType:
numEvents += rep['events']
return numEvents
## Extract the reports of the input files.
poolInOnlyRes = {}
for jobid, reports in reportData['runsAndLumis'].iteritems():
poolInOnlyRes[jobid] = [
rep for rep in reports if rep['type'] == 'POOLIN'
]
## Calculate how many input files have been processed.
numFilesProcessed = _getNumFiles(reportData['runsAndLumis'], 'POOLIN')
returndict['numFilesProcessed'] = numFilesProcessed
## Calculate how many events have been read.
numEventsRead = _getNumEvents(reportData['runsAndLumis'], 'POOLIN')
returndict['numEventsRead'] = numEventsRead
## Calculate how many events have been written.
numEventsWritten = {}
for filetype in ['EDM', 'TFile', 'FAKE']:
numEventsWritten[filetype] = _getNumEvents(
reportData['runsAndLumis'], filetype)
returndict['numEventsWritten'] = numEventsWritten
## Get the lumis in the input dataset.
returndict['inputDatasetLumis'] = reportData['inputDatasetLumis']
## Get the lumis split across files in the input dataset.
returndict['inputDatasetDuplicateLumis'] = reportData[
'inputDatasetDuplicateLumis']
## Get the lumis that the jobs had to process. This must be a subset of input
## dataset lumis & lumi-mask.
lumisToProcessPerJob = reportData['lumisToProcess']
lumisToProcess = {}
for jobid in lumisToProcessPerJob.keys():
for run, lumiRanges in lumisToProcessPerJob[jobid].iteritems():
if run not in lumisToProcess:
lumisToProcess[run] = []
for lumiRange in lumiRanges:
lumisToProcess[run].extend(
range(int(lumiRange[0]),
int(lumiRange[1]) + 1))
lumisToProcess = LumiList(runsAndLumis=lumisToProcess).getCompactList()
returndict['lumisToProcess'] = lumisToProcess
## Get the lumis that have been processed.
processedLumis = BasicJobType.mergeLumis(poolInOnlyRes)
returndict['processedLumis'] = processedLumis
outputDatasetsLumis = {}
outputDatasetsNumEvents = {}
if reportData['publication']:
## Get the run-lumi and number of events information about the output datasets.
outputDatasetsInfo = reportData['outputDatasetsInfo'][
'outputDatasets']
for dataset in outputDatasetsInfo:
if outputDatasetsInfo[dataset]['lumis']:
outputDatasetsLumis[dataset] = outputDatasetsInfo[dataset][
'lumis']
outputDatasetsNumEvents[dataset] = outputDatasetsInfo[dataset][
'numEvents']
returndict['outputDatasetsLumis'] = outputDatasetsLumis
returndict['outputDatasetsNumEvents'] = outputDatasetsNumEvents
numOutputDatasets = len(reportData['outputDatasetsInfo']
) if 'outputDatasetsInfo' in reportData else 0
## Get the duplicate runs-lumis in the output files. Use for this the run-lumi
## information of the input files. Why not to use directly the output files?
## Because not all types of output files have run-lumi information in their
## filemetadata (note: the run-lumi information in the filemetadata is a copy
## of the corresponding information in the FJR). For example, output files
## produced by TFileService do not have run-lumi information in the FJR. On the
## other hand, input files always have run-lumi information in the FJR, which
## lists the runs-lumis in the input file that have been processed by the
## corresponding job. And of course, the run-lumi information of an output file
## produced by job X should be the (set made out of the) union of the run-lumi
## information of the input files to job X.
outputFilesLumis = {}
for jobid, reports in poolInOnlyRes.iteritems():
if jobid.startswith('0-'): # skip probe-jobs
continue
lumiDict = {}
for rep in reports:
for run, lumis in literal_eval(rep['runlumi']).iteritems():
lumiDict.setdefault(str(run), []).extend(map(int, lumis))
for run, lumis in lumiDict.iteritems():
outputFilesLumis.setdefault(run, []).extend(list(set(lumis)))
outputFilesDuplicateLumis = BasicJobType.getDuplicateLumis(
outputFilesLumis)
returndict['outputFilesDuplicateLumis'] = outputFilesDuplicateLumis
## Calculate the not processed runs-lumis in one of three ways:
## 1) The lumis that were supposed to be processed by all jobs minus the lumis
## that were processed by finished (but not necessarily published) jobs.
## 2) The lumis that were supposed to be processed by all jobs minus the lumis
## published in all the output datasets.
## 3) The lumis that were supposed to be processed by jobs whose status is
## 'failed'.
notProcessedLumis = {}
notProcLumisCalcMethMsg = "The '%s' lumis were calculated as:" % (
self.options.recovery)
if self.options.recovery == 'notFinished':
notProcessedLumis = BasicJobType.subtractLumis(
lumisToProcess, processedLumis)
notProcLumisCalcMethMsg += " the lumis to process minus the processed lumis."
elif self.options.recovery == 'notPublished':
publishedLumis = {}
firstdataset = True
for dataset in outputDatasetsLumis.keys():
if firstdataset:
publishedLumis = outputDatasetsLumis[dataset]
firstdataset = False
else:
publishedLumis = BasicJobType.intersectLumis(
publishedLumis, outputDatasetsLumis[dataset])
notProcessedLumis = BasicJobType.subtractLumis(
lumisToProcess, publishedLumis)
notProcLumisCalcMethMsg += " the lumis to process"
if numOutputDatasets > 1:
notProcLumisCalcMethMsg += " minus the lumis published in all the output datasets."
else:
notProcLumisCalcMethMsg += " minus the lumis published in the output dataset."
elif self.options.recovery == 'failed':
for jobid, status in reportData['jobList']:
if status in ['failed']:
for run, lumiRanges in lumisToProcessPerJob[
jobid].iteritems():
if run not in notProcessedLumis:
notProcessedLumis[run] = []
for lumiRange in lumiRanges:
notProcessedLumis[run].extend(
range(lumiRange[0], lumiRange[1] + 1))
notProcessedLumis = LumiList(
runsAndLumis=notProcessedLumis).getCompactList()
notProcLumisCalcMethMsg += " the lumis to process by jobs in status 'failed'."
returndict['notProcessedLumis'] = notProcessedLumis
## Create the output directory if it doesn't exists.
if self.options.outdir:
jsonFileDir = self.options.outdir
else:
jsonFileDir = os.path.join(self.requestarea, 'results')
self.logger.info("Will save lumi files into output directory %s" %
(jsonFileDir))
if not os.path.exists(jsonFileDir):
self.logger.debug("Creating directory %s" % (jsonFileDir))
os.makedirs(jsonFileDir)
## Create the report JSON files and print a report summary:
## 1) First the summary that depends solely on successfully finished jobs (and
## other general information about the task, but not on failed/running jobs).
if not onlyDBSSummary:
self.logger.info("Summary from jobs in status 'finished':")
msg = " Number of files processed: %d" % (numFilesProcessed)
msg += "\n Number of events read: %d" % (numEventsRead)
msg += "\n Number of events written in EDM files: %d" % (
numEventsWritten.get('EDM', 0))
msg += "\n Number of events written in TFileService files: %d" % (
numEventsWritten.get('TFile', 0))
msg += "\n Number of events written in other type of files: %d" % (
numEventsWritten.get('FAKE', 0))
self.logger.info(msg)
if processedLumis:
with open(os.path.join(jsonFileDir, 'processedLumis.json'),
'w') as jsonFile:
json.dump(processedLumis, jsonFile)
jsonFile.write("\n")
self.logger.info(
" Processed lumis written to processedLumis.json")
if notProcessedLumis:
filename = self.options.recovery + "Lumis.json"
with open(os.path.join(jsonFileDir, filename),
'w') as jsonFile:
json.dump(notProcessedLumis, jsonFile)
jsonFile.write("\n")
self.logger.info(
" %sWarning%s: '%s' lumis written to %s" %
(colors.RED, colors.NORMAL, self.options.recovery,
filename))
self.logger.info(" %s" % (notProcLumisCalcMethMsg))
if outputFilesDuplicateLumis:
with open(
os.path.join(jsonFileDir,
'outputFilesDuplicateLumis.json'),
'w') as jsonFile:
json.dump(outputFilesDuplicateLumis, jsonFile)
jsonFile.write("\n")
self.logger.info(
" %sWarning%s: Duplicate lumis in output files written to outputFilesDuplicateLumis.json"
% (colors.RED, colors.NORMAL))
## 2) Then the summary about output datasets in DBS. For this, publication must
## be True and the output files must be publishable.
if reportData['publication'] and reportData['outputDatasets']:
if onlyDBSSummary:
self.logger.info(
"Will provide a short report with information found in DBS."
)
self.logger.info("Summary from output datasets in DBS:")
if outputDatasetsNumEvents:
msg = " Number of events:"
for dataset, numEvents in outputDatasetsNumEvents.iteritems():
msg += "\n %s: %d" % (dataset, numEvents)
self.logger.info(msg)
if outputDatasetsLumis:
with open(
os.path.join(jsonFileDir, 'outputDatasetsLumis.json'),
'w') as jsonFile:
json.dump(outputDatasetsLumis, jsonFile)
jsonFile.write("\n")
self.logger.info(
" Output datasets lumis written to outputDatasetsLumis.json"
)
## 3) Finally additional files that can be useful for debugging.
if reportData['inputDatasetLumis'] or reportData[
'inputDatasetDuplicateLumis'] or lumisToProcess:
self.logger.info("Additional report lumi files:")
if reportData['inputDatasetLumis']:
with open(os.path.join(jsonFileDir, 'inputDatasetLumis.json'),
'w') as jsonFile:
json.dump(reportData['inputDatasetLumis'], jsonFile)
jsonFile.write("\n")
self.logger.info(
" Input dataset lumis (from DBS, at task submission time) written to inputDatasetLumis.json"
)
if reportData['inputDatasetDuplicateLumis']:
with open(
os.path.join(jsonFileDir,
'inputDatasetDuplicateLumis.json'),
'w') as jsonFile:
json.dump(reportData['inputDatasetDuplicateLumis'], jsonFile)
jsonFile.write("\n")
self.logger.info(
" Input dataset duplicate lumis (from DBS, at task submission time) written to inputDatasetDuplicateLumis.json"
)
if lumisToProcess:
with open(os.path.join(jsonFileDir, 'lumisToProcess.json'),
'w') as jsonFile:
json.dump(lumisToProcess, jsonFile)
jsonFile.write("\n")
self.logger.info(
" Lumis to process written to lumisToProcess.json")
return returndict
def testURL(self):
URL = 'https://cms-service-dqm.web.cern.ch/cms-service-dqm/CAF/certification/Collisions12/8TeV/Reprocessing/Cert_190456-195530_8TeV_08Jun2012ReReco_Collisions12_JSON.txt'
ll = LumiList(url=URL)
self.assertTrue(len(ll) > 0)
def testGetLumiWhitelist(self):
"""
_testGetLumiWhitelist_
Verify that the ACDC whitelist generation code works correctly. We'll
add jobs with the following lumi info:
# Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12]
# Run 2, lumis [5, 6, 7], [10, 11, 12], [15]
# Run 3, lumis [20]
And should get out a whitelist that looks like this:
{"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]}
"""
dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc")
testFileA = File(lfn=makeUUID(), size=1024, events=1024)
testFileA.addRun(Run(1, 1, 2))
testFileB = File(lfn=makeUUID(), size=1024, events=1024)
testFileB.addRun(Run(1, 3))
testJobA = self.getMinimalJob()
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testFileC = File(lfn=makeUUID(), size=1024, events=1024)
testFileC.addRun(Run(1, 4, 6))
testJobB = self.getMinimalJob()
testJobB.addFile(testFileC)
testFileD = File(lfn=makeUUID(), size=1024, events=1024)
testFileD.addRun(Run(1, 7))
testJobC = self.getMinimalJob()
testJobC.addFile(testFileD)
testFileE = File(lfn=makeUUID(), size=1024, events=1024)
testFileE.addRun(Run(1, 11, 12))
testJobD = self.getMinimalJob()
testJobD.addFile(testFileE)
testFileF = File(lfn=makeUUID(), size=1024, events=1024)
testFileF.addRun(Run(2, 5, 6, 7))
testJobE = self.getMinimalJob()
testJobE.addFile(testFileF)
testFileG = File(lfn=makeUUID(), size=1024, events=1024)
testFileG.addRun(Run(2, 10, 11, 12))
testJobF = self.getMinimalJob()
testJobF.addFile(testFileG)
testFileH = File(lfn=makeUUID(), size=1024, events=1024)
testFileH.addRun(Run(2, 15))
testJobG = self.getMinimalJob()
testJobG.addFile(testFileH)
testFileI = File(lfn=makeUUID(), size=1024, events=1024)
testFileI.addRun(Run(3, 20))
testJobH = self.getMinimalJob()
testJobH.addFile(testFileI)
testFileJ = File(lfn=makeUUID(), size=1024, events=1024)
testFileJ.addRun(Run(1, 9))
testJobI = self.getMinimalJob()
testJobI.addFile(testFileJ)
dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE,
testJobF, testJobG, testJobH, testJobI])
whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(len(whiteList.keys()), 3,
"Error: There should be 3 runs.")
self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]],
"Error: Whitelist for run 1 is wrong.")
self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]],
"Error: Whitelist for run 2 is wrong.")
self.assertEqual(whiteList["3"], [[20, 20]],
"Error: Whitelist for run 3 is wrong.")
correctLumiList = LumiList(compactList={"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]})
testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(correctLumiList.getCMSSWString(), testLumiList.getCMSSWString())
return
def makeLumiList(lumiDict):
try:
ll = LumiList(compactList=lumiDict)
return ll.getCompactList()
except:
raise WMWorkloadToolsException("Could not parse LumiList")
def _handleInputDataInfo(self, wflow, dset, blocksDict):
"""
Applies any run/block/lumi list on the primary and parent
blocks provided.
It's a convoluted logic, such as:
1) if there is no run/block/lumi list, just return the initial blocksDict
2) if it has lumi list, filter runs from it and run block discovery
given a dataset name and a list of runs
3) if it has RunWhitelist, run block discovery for a given dataset name
and a list of runs
4) if it has only RunBlacklist, discover the run list of all initial blocks
provided in blocksDict and remove blocks matching only the black list
5) for the steps above, always check whether the block has replicas
6) NOW that the block data discovery is completed (considering runs):
* if LumiList is not enabled, just return the current list of blocks
* else, fetch file/run/lumi information in bulk of blocks and compare it
to the LumiList, skipping blocks without a single file that matches it.
Note that the LumiList check is dealt with in a similar way
as done in the WorkQueue StartPolicyInterface/getMaskedBlocks:
:param wflow: the Workflow object
:param dset: dataset name
:param blocksDict: dictionary of blocks, their size and location
:return: dictionary of block names and block size
"""
finalBlocks = {}
dbsUrl = wflow.getDbsUrl()
runWhite = wflow.getRunWhitelist()
runBlack = set(wflow.getRunBlacklist())
lumiList = wflow.getLumilist()
if lumiList:
# LumiList has precedence over RunWhitelist
runWhite = []
for run in lumiList.getRuns():
runWhite.append(int(run))
runWhite = list(set(runWhite))
if runWhite:
# Run number 1 is not supported by DBSServer
if int(runWhite[0]) == 1:
finalBlocks = deepcopy(blocksDict)
else:
runWhite = list(set(runWhite) - runBlack)
self.logger.info(
"Fetching blocks matching a list of runs for %s",
wflow.getName())
try:
blocks = getBlocksByDsetAndRun(dset, runWhite, dbsUrl)
except Exception as exc:
msg = "Failed to retrieve blocks by dataset '%s'and run: %s\n" % (
dset, runWhite)
msg += "Error details: %s" % str(exc)
self.logger.error(msg)
raise
for block in blocks:
if block in blocksDict:
finalBlocks[block] = deepcopy(blocksDict[block])
else:
self.logger.warning(
"Dropping block existent in DBS but not in Rucio: %s",
block)
elif runBlack:
# only run blacklist set
self.logger.info("Fetching runs in blocks for RunBlacklist for %s",
wflow.getName())
try:
blockRuns = getRunsInBlock(list(blocksDict), dbsUrl)
except Exception as exc:
self.logger.error(
"Failed to bulk retrieve runs per block. Details: %s",
str(exc))
raise
for block, runs in viewitems(blockRuns):
if not set(runs).difference(runBlack):
self.logger.info(
"Dropping block with only blacklisted runs: %s", block)
elif block in blocksDict:
finalBlocks[block] = deepcopy(blocksDict[block])
if lumiList:
self.logger.info(
"Fetching block/lumi information for %d blocks in %s",
len(finalBlocks), wflow.getName())
self.logger.debug("with the following run whitelist: %s", runWhite)
goodBlocks = set()
# now with a smaller set of blocks in hand, we collect their lumi
# information and discard any blocks not matching the lumi list
for blockSlice in grouper(finalBlocks, 10):
try:
blockFileLumis = getFileLumisInBlock(blockSlice,
dbsUrl,
validFileOnly=1)
except Exception as exc:
self.logger.error(
"Failed to bulk retrieve run/lumi per block. Details: %s",
str(exc))
raise
for block, fileLumis in viewitems(blockFileLumis):
for fileLumi in fileLumis:
if int(fileLumi['run_num']) not in runWhite:
continue
runNumber = str(fileLumi['run_num'])
lumis = fileLumi['lumi_section_num']
fileMask = LumiList(runsAndLumis={runNumber: lumis})
if lumiList & fileMask:
# then it has lumis that we need, keep this block and move on
goodBlocks.add(block)
break
# last but not least, drop any blocks that are not in the good list
for block in list(finalBlocks):
if block not in goodBlocks:
self.logger.info(
"Dropping block not matching LumiList: %s", block)
finalBlocks.pop(block)
if not finalBlocks:
finalBlocks = blocksDict
return self._removeZeroSizeBlocks(finalBlocks)
def algorithm(self, *args, **kwargs):
"""
_algorithm_
Split files into a number of lumis per job
Allow a flag to determine if we split files between jobs
"""
# TODO: This has one possible weakness. If two files (blocks) contain the same run/lumi section but are
# TODO: split across sites two different jobs will read the same lumi
avgEventsPerJob = int(kwargs.get('events_per_job', 5000))
jobTimeLimit = int(kwargs.get('job_time_limit', self.defaultJobTimeLimit))
totalEventLimit = int(kwargs.get('total_events', 0))
splitOnFile = bool(kwargs.get('halt_job_on_file_boundaries', False))
self.collectionName = kwargs.get('collectionName', None)
splitOnRun = kwargs.get('splitOnRun', True)
getParents = kwargs.get('include_parents', False)
runWhitelist = kwargs.get('runWhitelist', [])
runs = kwargs.get('runs', None)
lumis = kwargs.get('lumis', None)
self.deterministicPU = kwargs.get('deterministicPileup', False)
self.perfParameters = kwargs.get('performance', {})
# Calculate and add performance information
self.timePerEvent, self.sizePerEvent, self.memoryRequirement = \
self.getPerformanceParameters(self.perfParameters)
if avgEventsPerJob <= 0:
msg = "events_per_job parameter must be positive. Its value is: %d" % avgEventsPerJob
raise RuntimeError(msg)
# Set the lumi mask for the fileset based on ACDC or runs & lumis and/or runWhitelist
lumiMask = LumiList()
if self.collectionName:
lumiMask = self.lumiListFromACDC(couchURL=kwargs.get('couchURL'), couchDB=kwargs.get('couchDB'),
filesetName=kwargs.get('filesetName'), collectionName=self.collectionName)
elif runs and lumis and runWhitelist:
lumiMask = LumiList(wmagentFormat=(runs, lumis)) & LumiList(runs=runWhitelist)
elif runs and lumis:
lumiMask = LumiList(wmagentFormat=(runs, lumis))
elif runWhitelist:
lumiMask = LumiList(runs=runWhitelist)
logging.debug('%s splitting with lumiMask%s%s', self.__class__.__name__, '\n' if bool(lumiMask) else ' ',
lumiMask)
if self.package == 'WMCore.WMBS':
self.loadRunLumi = self.daoFactory(classname="Files.GetBulkRunLumi")
if self.deterministicPU:
getJobNumber = self.daoFactory(classname="Jobs.GetNumberOfJobsPerWorkflow")
self.nJobs = getJobNumber.execute(workflow=self.subscription.getWorkflow().id)
logging.info('Creating jobs in DeterministicPileup mode for %s',
self.subscription.workflowName())
filesByLocation = self.getFilesSortedByLocation(avgEventsPerJob)
if not filesByLocation:
logging.info("There are not enough events/files to be splitted. Trying again next cycle")
return
totalEvents = 0
lastRun = None
stopTask = False
lastFile = None
for location, filesAtLocation in viewitems(filesByLocation):
self.newGroup() # For each location, we need a new jobGroup
self.eventsInJob = 0
self.jobLumis = []
self.jobFiles = set()
self.lumisProcessed = set()
if self.loadRunLumi:
self.populateFilesFromWMBS(filesAtLocation)
lumisByFile, eventsByLumi = self.fileLumiMaps(filesAtLocation=filesAtLocation, getParents=getParents,
lumiMask=lumiMask)
for f in filesAtLocation:
lfn = f['lfn']
if lfn not in lumisByFile:
continue # There are no lumis of interest in the file
for run, lumi in lumisByFile[lfn].getLumis():
if (run, lumi) in self.lumisProcessed:
continue # We already saw this lumi and it got included in an earlier job
self.eventsInLumi = eventsByLumi[run][lumi]
if 0 < totalEventLimit <= totalEvents:
stopTask = True
break # Don't add this lumi to the job
totalEvents += self.eventsInLumi
self.maxEvents = max(self.maxEvents, self.eventsInLumi)
timePerLumi = self.eventsInLumi * self.timePerEvent
if timePerLumi > jobTimeLimit and len(lumisByFile[lfn].getLumis()):
# This lumi has too many events. Output this job and a new one with just that lumi
failReason = "File %s has a single lumi %s, in run %s " % (lfn, lumi, run)
failReason += "with too many events %d and it woud take %d sec to run" \
% (self.eventsInLumi, timePerLumi)
self.stopAndMakeJob(reason='Lumi too big', runLumi=(run, lumi),
failNextJob=True, failReason=failReason)
elif abs(self.eventsInLumi + self.eventsInJob - avgEventsPerJob) >= abs(
self.eventsInJob - avgEventsPerJob) \
and self.eventsInLumi > 0 and self.eventsInJob > 0:
# This lumi doesn't fit in this job (logic is to get as close as possible to avgEventsPerJob)
self.stopAndMakeJob(reason='Event limit', runLumi=(run, lumi))
elif splitOnRun and run != lastRun and self.eventsInJob > 0:
# This lumi is from a new run and we break on it
self.stopAndMakeJob(reason='Run change', runLumi=(run, lumi))
elif splitOnFile and self.filesByLumi[run][lumi][0]['lfn'] != lastFile and len(self.jobLumis):
# This lumi is from a new file and we break on it
self.stopAndMakeJob(reason='File change', runLumi=(run, lumi))
else: # Keep going, add lumi to list of lumis for the job
for f2 in self.filesByLumi[run][lumi]:
self.jobFiles.add(f2)
self.jobLumis.append((run, lumi))
self.eventsInJob += self.eventsInLumi
lastRun = run
lastFile = lfn
# Make the last job assuming its not just a failed job
if self.eventsInJob > 0 or self.jobLumis or self.jobFiles: # Make sure there is someting to do
self.makeNewJobByWork(reason='End of list')
if stopTask:
break
return
def subtractLumis(lumisA, lumisB):
result = LumiList(compactList=lumisA) - LumiList(compactList=lumisB)
return result.getCompactList()
def intersectLumis(lumisA, lumisB):
result = LumiList(compactList=lumisA) & LumiList(compactList=lumisB)
return result.getCompactList()
# now make the rest of the crab3 config
if isData:
config.Data.splitting = 'LumiBased' #LumiBased for data
if math.fabs(nUnitsPerJob)==1:
print 'WARNING: You specified +/-1 lumis per job; using default lumis per job of 100 instead'
config.Data.unitsPerJob = 40
config.Data.unitsPerJob = nUnitsPerJob
print 'INFO: using',nUnitsPerJob,'units (files/lumis) per job'
config.General.requestName = datasetName
config.JobType.psetName = newCmsswConfig
config.Data.totalUnits = nUnits
if options.jsonFile is not None:
if options.prevJsonFile is not None:
print 'Using the subtraction between previous json and new json; WARNING: if lumis changed from good in previous to bad in new json, this will not remove them'
from WMCore.DataStructs.LumiList import LumiList
prevJsonLumiList = LumiList(url=options.prevJsonFile) if 'http:' in options.prevJsonFile else LumiList(filename=options.prevJsonFile)
currentJsonLumiList = LumiList(url=options.jsonFile) if 'http:' in options.jsonFile else LumiList(filename=options.jsonFile)
newLumiList = currentJsonLumiList - prevJsonLumiList
newLumiList.writeJSON('newJSON_minus_oldJSON.json')
config.Data.lumiMask = 'newJSON_minus_oldJSON.json'
else:
print "USING JSON: ", options.jsonFile
config.Data.lumiMask = options.jsonFile
if options.runRange is not None:
config.Data.runRange = runRange
# and submit
print 'submit!'
#crabSubmit(config)
# workaround for cmssw multiple-loading problem
# submit in subprocess
q = Queue()
from CRABClient.UserUtilities import config, getLumiListInValidFiles
from WMCore.DataStructs.LumiList import LumiList
taskALumis = getLumiListInValidFiles(dataset='/HIMinimumBias6/qwang-V0Skim_v3-5f932986cf38f9e8dbd6c3aea7f6c2b4/USER', dbsurl='phys03')
officialLumiMask = LumiList(filename='/afs/cern.ch/cms/CAF/CMSCOMM/COMM_DQM/certification/Collisions18/HI/PromptReco/Cert_326381-327564_HI_PromptReco_Collisions18_JSON.txt')
newLumiMask = officialLumiMask - taskALumis
newLumiMask.writeJSON('t.json')
