def setUp(self):
"""
setup for test.
"""
super(ErrorHandlerTest, self).setUp()
myThread = threading.currentThread()
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMCore.WMBS"],
useDefault=False)
self.testInit.setupCouch("errorhandler_t", "GroupUser", "ACDC")
self.testInit.setupCouch("errorhandler_t_jd/jobs", "JobDump")
self.testInit.setupCouch("errorhandler_t_jd/fwjrs", "FWJRDump")
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.setJobTime = self.daofactory(classname="Jobs.SetStateTime")
locationAction = self.daofactory(classname="Locations.New")
locationAction.execute(siteName="malpaquet", pnn="T2_CH_CERN")
self.testDir = self.testInit.generateWorkDir()
self.configFile = EmulatorSetup.setupWMAgentConfig()
self.nJobs = 10
self.dataCS = DataCollectionService(url=self.testInit.couchUrl,
database="errorhandler_t")
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.changeState = ChangeState(self.config)
self.maxRetries = self.config.ErrorHandler.maxRetries
if not isinstance(self.maxRetries, dict):
self.maxRetries = {'default' : self.maxRetries}
if 'default' not in self.maxRetries:
raise ErrorHandlerException('Max retries for the default job type must be specified')
self.maxProcessSize = getattr(self.config.ErrorHandler, 'maxProcessSize', 250)
self.exitCodes = getattr(self.config.ErrorHandler, 'failureExitCodes', [])
self.maxFailTime = getattr(self.config.ErrorHandler, 'maxFailTime', 32 * 3600)
self.readFWJR = getattr(self.config.ErrorHandler, 'readFWJR', False)
self.passCodes = getattr(self.config.ErrorHandler, 'passExitCodes', [])
self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")
self.idLoad = self.daoFactory(classname = "Jobs.LoadFromIDWithType")
self.loadAction = self.daoFactory(classname = "Jobs.LoadForErrorHandler")
self.dataCollection = DataCollectionService(url = config.ACDC.couchurl,
database = config.ACDC.database)
return
def validBlocks(self, task):
"""Return blocks that pass the input data restriction"""
validBlocks = []
# TODO take the chunk size from parameter
chunkSize = 200
acdcInfo = task.getInputACDC()
if not acdcInfo:
raise WorkQueueWMSpecError(
self.wmspec, 'No acdc section for %s' % task.getPathName())
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
if self.data:
acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0])
else:
#if self.data is not passed, assume the the data is input dataset
# from the spec
acdcBlockSplit = False
if acdcBlockSplit:
dbsBlock = {}
dbsBlock['Name'] = self.data.keys()[0]
block = acdc.getChunkInfo(
acdcInfo['collection'],
acdcBlockSplit['TaskName'],
acdcBlockSplit['Offset'],
acdcBlockSplit['NumOfFiles'],
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock['ACDC'] = acdcInfo
dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"])
validBlocks.append(dbsBlock)
else:
acdcBlocks = acdc.chunkFileset(
acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize,
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'],
block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock["Sites"] = sitesFromStorageEelements(
block["locations"])
dbsBlock['ACDC'] = acdcInfo
validBlocks.append(dbsBlock)
return validBlocks
def testFailedJobsUniqueWf(self):
"""
Performance test of failedJobs with all failed jobs belonging
to the same workflow and the same task name
"""
loadList = []
for i in range(1, 5000):
loadList.append(self.jobConfig('wf1', '/wf1/task1', i, 'lfn1'))
dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc")
dcs.failedJobs(loadList)
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
Split files into a number of lumis per job
Allow a flag to determine if we split files between jobs
"""
myThread = threading.currentThread()
lumisPerJob = int(kwargs.get('lumis_per_job', 1))
splitOnFile = bool(kwargs.get('halt_job_on_file_boundaries', True))
ignoreACDC = bool(kwargs.get('ignore_acdc_except', False))
collectionName = kwargs.get('collectionName', None)
splitOnRun = kwargs.get('splitOnRun', True)
getParents = kwargs.get('include_parents', False)
runWhitelist = kwargs.get('runWhitelist', [])
runs = kwargs.get('runs', None)
lumis = kwargs.get('lumis', None)
goodRunList = {}
if runs and lumis:
goodRunList = buildLumiMask(runs, lumis)
# If we have runLumi info, we need to load it from couch
if collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
collectionName = kwargs.get('collectionName')
owner = kwargs.get('owner')
group = kwargs.get('group')
logging.info('Creating jobs for ACDC fileset %s' % filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumiWhitelist(collectionName, filesetName,
owner, group)
except Exception, ex:
msg = "Exception while trying to load goodRunList\n"
if ignoreACDC:
msg += "Ditching goodRunList\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
goodRunList = {}
else:
msg += "Refusing to create any jobs.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
return
def validBlocks(self, task):
"""Return blocks that pass the input data restriction according
to the splitting algorithm"""
validBlocks = []
acdcInfo = task.getInputACDC()
if not acdcInfo:
raise WorkQueueWMSpecError(
self.wmspec, 'No acdc section for %s' % task.getPathName())
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
if self.data:
acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0])
else:
# if self.data is not passed, assume the the data is input dataset
# from the spec
acdcBlockSplit = False
if acdcBlockSplit:
dbsBlock = {}
dbsBlock['Name'] = self.data.keys()[0]
block = acdc.getChunkInfo(
acdcInfo['collection'],
acdcBlockSplit['TaskName'],
acdcBlockSplit['Offset'],
acdcBlockSplit['NumOfFiles'],
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock['ACDC'] = acdcInfo
if task.getTrustSitelists():
dbsBlock["Sites"] = self.sites
else:
# TODO remove this line when all DBS origin_site_name is converted to PNN
block["locations"] = self.siteDB.checkAndConvertSENameToPNN(
block["locations"])
# upto this
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"])
validBlocks.append(dbsBlock)
else:
if self.args['SplittingAlgo'] in self.unsupportedAlgos:
raise WorkQueueWMSpecError(
self.wmspec, 'ACDC is not supported for %s' %
self.args['SplittingAlgo'])
splittingFunc = self.defaultAlgo
if self.args['SplittingAlgo'] in self.algoMapping:
splittingFunc = self.algoMapping[self.args['SplittingAlgo']]
validBlocks = splittingFunc(acdc, acdcInfo, task)
return validBlocks
def main():
start = time.time()
# blockName = match['Inputs'].keys()[0]
blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055"
# acdcInfo = match['ACDC']
acdcInfo = {"database": "acdcserver",
"fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0",
"collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222",
"server": "https://cmsweb.cern.ch/couchdb"}
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
splitedBlockName = ACDCBlock.splitBlockName(blockName)
print("Splitted block name: %s" % splitedBlockName)
fileLists = acdc.getChunkFiles(acdcInfo['collection'],
acdcInfo['fileset'],
splitedBlockName['Offset'],
splitedBlockName['NumOfFiles'])
print("Retrieved %d unique files from the ACDCServer" % len(fileLists))
block = {}
block["Files"] = fileLists
wantedLumis = set([252052, 240646])
for f in fileLists:
for run in f['runs']:
maskDict = run.json()
lumisSet = set(maskDict['Lumis'].keys())
if wantedLumis.intersection(lumisSet):
print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet)))
# with open("chunkfiles.json", 'w') as fo:
# json.dump(block, fo)
end = time.time()
print("Spent %s secs running so far" % (end - start))
sys.exit(1)
### Now doing the WMBSHelper stuff
reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"
requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376"
wmspec = WMWorkloadHelper()
wmspec.loadSpecFromCouch(reqUrl, requestName)
taskName = "HIG-RunIIFall17DRPremix-00788_0"
mask = None
cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache"
# wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir'])
wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir)
sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def testFailedJobsScrambledWf(self):
"""
Performance test of failedJobs where jobs belong to 10 different
workflows and 3 different tasks
"""
loadList = []
for i in range(1, 5000):
wfName = "wf%d" % (i % 10)
taskName = "/wf%d/task%d" % (i % 10, i % 3)
loadList.append(self.jobConfig(wfName, taskName, i, '/file/name/lfn1'))
dcs = DataCollectionService(url=self.testInit.couchUrl, database="wmcore-acdc-datacollectionsvc")
dcs.failedJobs(loadList)
return
def main():
start = time.time()
# acdcInfo = match['ACDC']
acdcInfo = {
"database": "acdcserver",
"fileset":
"/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0",
"collection":
"pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222",
"server": "https://cmsweb.cern.ch/couchdb"
}
dcs = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
# acdcFileList = dcs.getProductionACDCInfo(acdcInfo['collection'], acdcInfo['fileset'])
files = dcs._getFilesetInfo(acdcInfo['collection'], acdcInfo['fileset'])
print("%s" % pformat(files[0]))
files = mergeFilesInfo(files)
acdcFileList = []
for value in files:
fileInfo = {
"lfn": value["lfn"],
"first_event": value["first_event"],
"lumis": value["runs"][0]["lumis"],
"events": value["events"]
}
acdcFileList.append(fileInfo)
#print("Data retrieved:\n%s" % pformat(acdcFileList))
print("Retrieved %d files from the ACDCServer" % len(acdcFileList))
listLumis = []
wantedLumis = set([252052, 240646])
for f in acdcFileList:
listLumis.extend(f['lumis'])
lumisSet = set(f['lumis'])
if wantedLumis.intersection(lumisSet):
print("File: %s with events: %s, contains these lumis: %s" %
(f['lfn'], f['events'], f['lumis']))
print("Total amount of lumis: %d, where unique are: %d" %
(len(listLumis), len(set(listLumis))))
# with open("chunkfiles.json", 'w') as fo:
# json.dump(block, fo)
end = time.time()
print("Spent %s secs running so far" % (end - start))
sys.exit(1)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.changeState = ChangeState(self.config)
self.maxRetries = self.config.ErrorHandler.maxRetries
if type(self.maxRetries) != dict:
self.maxRetries = {'default': self.maxRetries}
if 'default' not in self.maxRetries:
raise ErrorHandlerException(
'Max retries for the default job type must be specified')
self.maxProcessSize = getattr(self.config.ErrorHandler,
'maxProcessSize', 250)
self.exitCodes = getattr(self.config.ErrorHandler, 'failureExitCodes',
[])
self.maxFailTime = getattr(self.config.ErrorHandler, 'maxFailTime',
32 * 3600)
self.readFWJR = getattr(self.config.ErrorHandler, 'readFWJR', False)
self.passCodes = getattr(self.config.ErrorHandler, 'passExitCodes', [])
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
self.idLoad = self.daoFactory(classname="Jobs.LoadFromIDWithType")
self.loadAction = self.daoFactory(classname="Jobs.LoadForErrorHandler")
self.dataCollection = DataCollectionService(
url=config.ACDC.couchurl, database=config.ACDC.database)
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName="ErrorHandler")
# Some exit codes imply an immediate failure, non-configurable
self.exitCodes.extend(WMJobPermanentSystemErrors)
return
def lumiListFromACDC(couchURL=None, couchDB=None, filesetName=None, collectionName=None):
"""
This is not implemented yet
:return:
"""
from WMCore.ACDC.DataCollectionService import DataCollectionService
goodRunList = None
try:
logging.info('Creating jobs for ACDC fileset %s', filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumilistWhitelist(collectionName, filesetName)
except Exception as ex:
msg = "Exception while trying to load goodRunList. "
msg += "Refusing to create any jobs.\nDetails: %s" % ex.__str__()
logging.exception(msg)
return goodRunList
def validBlocks(self, task):
"""Return blocks that pass the input data restriction according
to the splitting algorithm"""
validBlocks = []
acdcInfo = task.getInputACDC()
if not acdcInfo:
raise WorkQueueWMSpecError(
self.wmspec, 'No acdc section for %s' % task.getPathName())
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
if self.data:
acdcBlockSplit = ACDCBlock.splitBlockName(next(iter(self.data)))
else:
# if self.data is not passed, assume the the data is input dataset
# from the spec
acdcBlockSplit = False
if acdcBlockSplit:
dbsBlock = {}
dbsBlock['Name'] = next(iter(self.data))
block = acdc.getChunkInfo(acdcInfo['collection'],
acdcBlockSplit['TaskName'],
acdcBlockSplit['Offset'],
acdcBlockSplit['NumOfFiles'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock['ACDC'] = acdcInfo
if task.getTrustSitelists().get('trustlists'):
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = self.cric.PNNstoPSNs(block["locations"])
validBlocks.append(dbsBlock)
else:
if self.args['SplittingAlgo'] in self.unsupportedAlgos:
raise WorkQueueWMSpecError(
self.wmspec, 'ACDC is not supported for %s' %
self.args['SplittingAlgo'])
splittingFunc = self.defaultAlgo
if self.args['SplittingAlgo'] in self.algoMapping:
splittingFunc = self.algoMapping[self.args['SplittingAlgo']]
validBlocks = splittingFunc(acdc, acdcInfo, task)
return validBlocks
def lumiListFromACDC(couchURL=None, couchDB=None, filesetName=None, collectionName=None, ignoreACDC=False):
"""
This is not implemented yet
:return:
"""
try:
logging.info('Creating jobs for ACDC fileset %s', filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumilistWhitelist(collectionName, filesetName)
except Exception as ex:
msg = "Exception while trying to load goodRunList\n"
if ignoreACDC: # Logic can go in main function?
msg += "Ditching goodRunList\n" + str(ex) + str(traceback.format_exc())
logging.error(msg)
goodRunList = {}
else:
msg += "Refusing to create any jobs.\n" + str(ex) + str(traceback.format_exc())
logging.error(msg)
return None # An error condtion - check
return goodRunList
def test06UploadACDC(self):
# get previous request we can piggyback on
for request in reversed(self.__class__.reqmgr.getRequest()):
request = request[
'WMCore.RequestManager.DataStructs.Request.Request'][
'RequestName']
if 'RequestCancellation_t' in request:
self.__class__.requestParams['OriginalRequestName'] = request
break
else:
raise nose.SkipTest("no suitable request in reqmgr to resubmit")
self.__class__.requestParams[
'InitialTaskPath'] = self.__class__.requestParams[
'InitialTaskPath'] % self.__class__.requestParams[
'OriginalRequestName']
self.__class__.requestParams[
'ACDCServer'] = self.__class__.endpoint + '/couchdb'
# create and upload acdc
service = DataCollectionService(url=self.__class__.endpoint +
'/couchdb',
database='wmagent_acdc')
service.createCollection(
self.__class__.requestParams['OriginalRequestName'], 'integration',
'DMWM')
with open(
os.path.join(
getTestBase(), '..', 'data', 'ACDC',
'linacre_ACDC_ReReco13JulCosmics_120809_130020_117_120823_200309_5735.json'
)) as infile:
acdc_json = infile.read().replace(
'linacre_ACDC_ReReco13JulCosmics_120809_130020_117_120823_200309_5735',
self.__class__.requestParams['OriginalRequestName'])
acdc_json = loads(acdc_json)
acdc_database = Database('wmagent_acdc',
self.__class__.endpoint + '/couchdb')
acdc_database.commit(acdc_json)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.changeState = ChangeState(self.config)
if hasattr(self.config, "Tier0Feeder"):
self.reqAuxDB = None
self.maxRetries = self.config.ErrorHandler.maxRetries
else:
self.reqAuxDB = ReqMgrAux(self.config.General.ReqMgr2ServiceURL)
self.exitCodesNoRetry = []
self.maxProcessSize = getattr(self.config.ErrorHandler,
'maxProcessSize', 250)
self.maxFailTime = getattr(self.config.ErrorHandler, 'maxFailTime',
32 * 3600)
self.readFWJR = getattr(self.config.ErrorHandler, 'readFWJR', False)
self.passCodes = getattr(self.config.ErrorHandler, 'passExitCodes', [])
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
self.idLoad = self.daoFactory(classname="Jobs.LoadFromIDWithType")
self.loadAction = self.daoFactory(classname="Jobs.LoadForErrorHandler")
self.dataCollection = DataCollectionService(
url=config.ACDC.couchurl, database=config.ACDC.database)
self.setupComponentParam()
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
An event base splitting algorithm. All available files are split into a
set number of events per job.
"""
eventsPerJob = int(kwargs.get("events_per_job", 100))
eventsPerLumi = int(kwargs.get("events_per_lumi", eventsPerJob))
getParents = kwargs.get("include_parents", False)
lheInput = kwargs.get("lheInputFiles", False)
collectionName = kwargs.get('collectionName', None)
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
acdcFileList = []
# If we have runLumi info, we need to load it from couch
if collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
collectionName = kwargs.get('collectionName')
owner = kwargs.get('owner')
group = kwargs.get('group')
logging.info('Creating jobs for ACDC fileset %s' % filesetName)
dcs = DataCollectionService(couchURL, couchDB)
acdcFileList = dcs.getProductionACDCInfo(
collectionName, filesetName, owner, group)
except Exception, ex:
msg = "Exception while trying to load goodRunList\n"
msg += "Refusing to create any jobs.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
return
def _getDBSBlock(self, match, wmspec):
"""Get DBS info for this block"""
blockName = match['Inputs'].keys()[0] #TODO: Allow more than one
if match['ACDC']:
acdcInfo = match['ACDC']
acdc = DataCollectionService(acdcInfo["server"],
acdcInfo["database"])
collection = acdc.getDataCollection(acdcInfo['collection'])
splitedBlockName = ACDCBlock.splitBlockName(blockName)
fileLists = acdc.getChunkFiles(
acdcInfo['collection'],
acdcInfo['fileset'],
splitedBlockName['Offset'],
splitedBlockName['NumOfFiles'],
user=wmspec.getOwner().get("name"),
group=wmspec.getOwner().get("group"))
block = {}
block["Files"] = fileLists
return blockName, block
else:
dbs = get_dbs(match['Dbs'])
if wmspec.getTask(match['TaskName']).parentProcessingFlag():
dbsBlockDict = dbs.getFileBlockWithParents(blockName)
else:
dbsBlockDict = dbs.getFileBlock(blockName)
if wmspec.locationDataSourceFlag():
blockInfo = dbsBlockDict[blockName]
seElements = []
for cmsSite in match['Inputs'].values(
)[0]: #TODO: Allow more than one
ses = self.SiteDB.cmsNametoSE(cmsSite)
seElements.extend(ses)
seElements = list(set(seElements))
blockInfo['StorageElements'] = seElements
return blockName, dbsBlockDict[blockName]
def testGetLumiWhitelist(self):
"""
_testGetLumiWhitelist_
Verify that the ACDC whitelist generation code works correctly. We'll
add jobs with the following lumi info:
# Run 1, lumis [1, 2, 3], [4, 6], [7], [9], [11, 12]
# Run 2, lumis [5, 6, 7], [10, 11, 12], [15]
# Run 3, lumis [20]
And should get out a whitelist that looks like this:
{"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]}
"""
dcs = DataCollectionService(url=self.testInit.couchUrl,
database="wmcore-acdc-datacollectionsvc")
testFileA = File(lfn=makeUUID(), size=1024, events=1024)
testFileA.addRun(Run(1, 1, 2))
testFileB = File(lfn=makeUUID(), size=1024, events=1024)
testFileB.addRun(Run(1, 3))
testJobA = self.getMinimalJob()
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testFileC = File(lfn=makeUUID(), size=1024, events=1024)
testFileC.addRun(Run(1, 4, 6))
testJobB = self.getMinimalJob()
testJobB.addFile(testFileC)
testFileD = File(lfn=makeUUID(), size=1024, events=1024)
testFileD.addRun(Run(1, 7))
testJobC = self.getMinimalJob()
testJobC.addFile(testFileD)
testFileE = File(lfn=makeUUID(), size=1024, events=1024)
testFileE.addRun(Run(1, 11, 12))
testJobD = self.getMinimalJob()
testJobD.addFile(testFileE)
testFileF = File(lfn=makeUUID(), size=1024, events=1024)
testFileF.addRun(Run(2, 5, 6, 7))
testJobE = self.getMinimalJob()
testJobE.addFile(testFileF)
testFileG = File(lfn=makeUUID(), size=1024, events=1024)
testFileG.addRun(Run(2, 10, 11, 12))
testJobF = self.getMinimalJob()
testJobF.addFile(testFileG)
testFileH = File(lfn=makeUUID(), size=1024, events=1024)
testFileH.addRun(Run(2, 15))
testJobG = self.getMinimalJob()
testJobG.addFile(testFileH)
testFileI = File(lfn=makeUUID(), size=1024, events=1024)
testFileI.addRun(Run(3, 20))
testJobH = self.getMinimalJob()
testJobH.addFile(testFileI)
testFileJ = File(lfn=makeUUID(), size=1024, events=1024)
testFileJ.addRun(Run(1, 9))
testJobI = self.getMinimalJob()
testJobI.addFile(testFileJ)
dcs.failedJobs([
testJobA, testJobB, testJobC, testJobD, testJobE, testJobF,
testJobG, testJobH, testJobI
])
whiteList = dcs.getLumiWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(len(whiteList.keys()), 3,
"Error: There should be 3 runs.")
self.assertEqual(whiteList["1"], [[1, 4], [6, 7], [9, 9], [11, 12]],
"Error: Whitelist for run 1 is wrong.")
self.assertEqual(whiteList["2"], [[5, 7], [10, 12], [15, 15]],
"Error: Whitelist for run 2 is wrong.")
self.assertEqual(whiteList["3"], [[20, 20]],
"Error: Whitelist for run 3 is wrong.")
correctLumiList = LumiList(
compactList={
"1": [[1, 4], [6, 7], [9, 9], [11, 12]],
"2": [[5, 7], [10, 12], [15, 15]],
"3": [[20, 20]]
})
testLumiList = dcs.getLumilistWhitelist("ACDCTest", "/ACDCTest/reco")
self.assertEqual(correctLumiList.getCMSSWString(),
testLumiList.getCMSSWString())
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
Split files into a number of lumis per job
Allow a flag to determine if we split files between jobs
"""
avgEventsPerJob = int(kwargs.get('events_per_job', 5000))
eventLimit = int(kwargs.get('max_events_per_lumi', 20000))
totalEvents = int(kwargs.get('total_events', 0))
splitOnFile = bool(kwargs.get('halt_job_on_file_boundaries', True))
ignoreACDC = bool(kwargs.get('ignore_acdc_except', False))
collectionName = kwargs.get('collectionName', None)
splitOnRun = kwargs.get('splitOnRun', True)
getParents = kwargs.get('include_parents', False)
runWhitelist = kwargs.get('runWhitelist', [])
runs = kwargs.get('runs', None)
lumis = kwargs.get('lumis', None)
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
capJobTime = kwargs.get('capJobTime', None)
capJobDisk = kwargs.get('capJobDisk', None)
deterministicPileup = kwargs.get('deterministicPileup', False)
eventsPerLumiInDataset = 0
if deterministicPileup and self.package == 'WMCore.WMBS':
getJobNumber = self.daoFactory(classname = "Jobs.GetNumberOfJobsPerWorkflow")
jobNumber = getJobNumber.execute(workflow = self.subscription.getWorkflow().id)
self.nJobs = jobNumber
goodRunList = {}
if runs and lumis:
goodRunList = buildLumiMask(runs, lumis)
# If we have runLumi info, we need to load it from couch
if collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
collectionName = kwargs.get('collectionName')
owner = kwargs.get('owner')
group = kwargs.get('group')
logging.info('Creating jobs for ACDC fileset %s' % filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumiWhitelist(collectionName, filesetName, owner, group)
except Exception as ex:
msg = "Exception while trying to load goodRunList\n"
if ignoreACDC:
msg += "Ditching goodRunList\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
goodRunList = {}
else:
msg += "Refusing to create any jobs.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
return
lDict = self.sortByLocation()
locationDict = {}
# First we need to load the data
if self.package == 'WMCore.WMBS':
loadRunLumi = self.daoFactory(classname = "Files.GetBulkRunLumi")
for key in lDict.keys():
newlist = []
# First we need to load the data
if self.package == 'WMCore.WMBS':
fileLumis = loadRunLumi.execute(files = lDict[key])
for f in lDict[key]:
lumiDict = fileLumis.get(f['id'], {})
for run in lumiDict.keys():
f.addRun(run = Run(run, *lumiDict[run]))
for f in lDict[key]:
if len(f['runs']) == 0:
continue
f['runs'] = sorted(f['runs'])
f['lumiCount'] = 0
for run in f['runs']:
run.lumis.sort()
f['lumiCount'] += len(run.lumis)
f['lowestRun'] = f['runs'][0]
#Do average event per lumi calculation
if f['lumiCount']:
f['avgEvtsPerLumi'] = round(float(f['events'])/f['lumiCount'])
if deterministicPileup:
# We assume that all lumis are equal in the dataset
eventsPerLumiInDataset = f['avgEvtsPerLumi']
else:
#No lumis in the file, ignore it
continue
newlist.append(f)
locationDict[key] = sorted(newlist, key=operator.itemgetter('lowestRun'))
totalJobs = 0
lastLumi = None
firstLumi = None
lastRun = None
lumisInJob = 0
totalAvgEventCount = 0
currentJobAvgEventCount = 0
stopTask = False
for location in locationDict:
# For each location, we need a new jobGroup
self.newGroup()
stopJob = True
for f in locationDict[location]:
if getParents:
parentLFNs = self.findParent(lfn = f['lfn'])
for lfn in parentLFNs:
parent = File(lfn = lfn)
f['parents'].add(parent)
lumisInJobInFile = 0
updateSplitOnJobStop = False
failNextJob = False
#If the number of events per lumi is higher than the limit
#and it's only one lumi then ditch that lumi
if f['avgEvtsPerLumi'] > eventLimit and f['lumiCount'] == 1:
failNextJob = True
stopJob = True
lumisPerJob = 1
elif splitOnFile:
# Then we have to split on every boundary
stopJob = True
#Check the average number of events per lumi in this file
#Adapt the lumis per job to match the target conditions
if f['avgEvtsPerLumi']:
#If there are events in the file
ratio = float(avgEventsPerJob) / f['avgEvtsPerLumi']
lumisPerJob = max(int(math.floor(ratio)), 1)
else:
#Zero event file, then the ratio goes to infinity. Computers don't like that
lumisPerJob = f['lumiCount']
else:
#Analyze how many events does this job already has
#Check how many we want as target, include as many lumi sections as possible
updateSplitOnJobStop = True
eventsRemaining = max(avgEventsPerJob - currentJobAvgEventCount, 0)
if f['avgEvtsPerLumi']:
lumisAllowed = int(math.floor(float(eventsRemaining) / f['avgEvtsPerLumi']))
else:
lumisAllowed = f['lumiCount']
lumisPerJob = max(lumisInJob + lumisAllowed, 1)
for run in f['runs']:
if not isGoodRun(goodRunList = goodRunList, run = run.run):
# Then skip this one
continue
if len(runWhitelist) > 0 and not run.run in runWhitelist:
# Skip due to run whitelist
continue
firstLumi = None
if splitOnRun and run.run != lastRun:
# Then we need to kill this job and get a new one
stopJob = True
# Now loop over the lumis
for lumi in run:
if not isGoodLumi(goodRunList, run = run.run, lumi = lumi):
# Kill the chain of good lumis
# Skip this lumi
if firstLumi != None and firstLumi != lumi:
self.currentJob['mask'].addRunAndLumis(run = run.run,
lumis = [firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
if capJobTime or capJobDisk:
self.currentJob.capResourceEstimates(jobTime = capJobTime, disk = capJobDisk)
firstLumi = None
lastLumi = None
continue
# You have to kill the lumi chain if they're not continuous
if lastLumi and not lumi == lastLumi + 1:
self.currentJob['mask'].addRunAndLumis(run = run.run,
lumis = [firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
if capJobTime or capJobDisk:
self.currentJob.capResourceEstimates(jobTime = capJobTime, disk = capJobDisk)
firstLumi = None
lastLumi = None
if firstLumi == None:
# Set the first lumi in the run
firstLumi = lumi
# If we're full, end the job
if lumisInJob == lumisPerJob:
stopJob = True
# Actually do the new job creation
if stopJob:
if firstLumi != None and lastLumi != None and lastRun != None:
self.currentJob['mask'].addRunAndLumis(run = lastRun,
lumis = [firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
if capJobTime or capJobDisk:
self.currentJob.capResourceEstimates(jobTime = capJobTime, disk = capJobDisk)
msg = None
if failNextJob:
msg = "File %s has too many events (%d) in %d lumi(s)" % (f['lfn'],
f['events'],
f['lumiCount'])
self.newJob(name = self.getJobName(), failedJob = failNextJob,
failedReason = msg)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * lumisPerJob * eventsPerLumiInDataset)
self.currentJob.addResourceEstimates(memory = memoryRequirement)
failNextJob = False
firstLumi = lumi
lumisInJob = 0
lumisInJobInFile = 0
currentJobAvgEventCount = 0
totalJobs += 1
# Add the file to new jobs
self.currentJob.addFile(f)
if updateSplitOnJobStop:
#Then we were carrying from a previous file
#Reset calculations for this file
updateSplitOnJobStop = False
if f['avgEvtsPerLumi']:
ratio = float(avgEventsPerJob) / f['avgEvtsPerLumi']
lumisPerJob = max(int(math.floor(ratio)), 1)
else:
lumisPerJob = f['lumiCount']
lumisInJob += 1
lumisInJobInFile += 1
lastLumi = lumi
stopJob = False
lastRun = run.run
totalAvgEventCount += f['avgEvtsPerLumi']
if self.currentJob and not f in self.currentJob['input_files']:
self.currentJob.addFile(f)
# We stop here if there are more total events than requested.
if totalEvents > 0 and totalAvgEventCount >= totalEvents:
stopTask = True
break
if firstLumi != None and lastLumi != None:
# Add this run to the mask
self.currentJob['mask'].addRunAndLumis(run = run.run,
lumis = [firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) * f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(jobTime = runAddedTime, disk = runAddedSize)
if capJobTime or capJobDisk:
self.currentJob.capResourceEstimates(jobTime = capJobTime, disk = capJobDisk)
firstLumi = None
lastLumi = None
if stopTask:
break
if not splitOnFile:
currentJobAvgEventCount += f['avgEvtsPerLumi'] * lumisInJobInFile
if stopTask:
break
if stopTask:
break
return
def testC_ACDCTest(self):
"""
_ACDCTest_
Test whether we can get a goodRunList out of ACDC
and process it correctly.
"""
workload = self.createTestWorkload()
dcs = DataCollectionService(url=self.testInit.couchUrl,
database=self.testInit.couchDbName)
testFileA = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileA.addRun(Run(1, 1, 2))
testFileA.create()
testFileB = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileB.addRun(Run(1, 3))
testFileB.create()
testJobA = getJob(workload)
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testFileC = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileC.addRun(Run(1, 4, 6))
testFileC.create()
testJobB = getJob(workload)
testJobB.addFile(testFileC)
testFileD = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileD.addRun(Run(1, 7))
testFileD.create()
testJobC = getJob(workload)
testJobC.addFile(testFileD)
testFileE = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileE.addRun(Run(1, 11, 12))
testFileE.create()
testJobD = getJob(workload)
testJobD.addFile(testFileE)
testFileF = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileF.addRun(Run(2, 5, 6, 7))
testFileF.create()
testJobE = getJob(workload)
testJobE.addFile(testFileF)
testFileG = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileG.addRun(Run(2, 10, 11, 12))
testFileG.create()
testJobF = getJob(workload)
testJobF.addFile(testFileG)
testFileH = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileH.addRun(Run(2, 15))
testFileH.create()
testJobG = getJob(workload)
testJobG.addFile(testFileH)
testFileI = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileI.addRun(Run(3, 20))
testFileI.create()
testJobH = getJob(workload)
testJobH.addFile(testFileI)
testFileJ = File(lfn=makeUUID(),
size=1024,
events=1024,
locations="T1_US_FNAL_Disk")
testFileJ.addRun(Run(1, 9))
testFileJ.create()
testJobI = getJob(workload)
testJobI.addFile(testFileJ)
# dcs.failedJobs([testJobA, testJobB, testJobC, testJobD, testJobE,
# testJobF, testJobG, testJobH, testJobI])
dcs.failedJobs([testJobA, testJobD, testJobH])
baseName = makeUUID()
testFileset = Fileset(name=baseName)
testFileset.create()
testFileset.addFile(testFileA)
testFileset.addFile(testFileB)
testFileset.addFile(testFileC)
testFileset.addFile(testFileD)
testFileset.addFile(testFileE)
testFileset.addFile(testFileF)
testFileset.addFile(testFileG)
testFileset.addFile(testFileH)
testFileset.addFile(testFileI)
testFileset.addFile(testFileJ)
testFileset.commit()
testSubscription = Subscription(fileset=testFileset,
workflow=self.testWorkflow,
split_algo="LumiBased",
type="Processing")
testSubscription.create()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=testSubscription)
jobGroups = jobFactory(
lumis_per_job=100,
halt_job_on_file_boundaries=False,
splitOnRun=True,
collectionName=workload.name(),
filesetName=workload.getTask("reco").getPathName(),
owner="evansde77",
group="DMWM",
couchURL=self.testInit.couchUrl,
couchDB=self.testInit.couchDbName,
performance=self.performanceParams)
self.assertEqual(jobGroups[0].jobs[0]['mask'].getRunAndLumis(),
{1: [[1, 2], [3, 3], [11, 12]]})
self.assertEqual(jobGroups[0].jobs[1]['mask'].getRunAndLumis(),
{3: [[20, 20]]})
return
def __init__(self, config):
"""
__init__
Create all DAO objects that are used by this class.
"""
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getOutputMapAction = self.daofactory(
classname="Jobs.GetOutputMap")
self.bulkAddToFilesetAction = self.daofactory(
classname="Fileset.BulkAddByLFN")
self.bulkParentageAction = self.daofactory(
classname="Files.AddBulkParentage")
self.getJobTypeAction = self.daofactory(classname="Jobs.GetType")
self.getParentInfoAction = self.daofactory(
classname="Files.GetParentInfo")
self.setParentageByJob = self.daofactory(
classname="Files.SetParentageByJob")
self.setParentageByMergeJob = self.daofactory(
classname="Files.SetParentageByMergeJob")
self.setFileRunLumi = self.daofactory(classname="Files.AddRunLumi")
self.setFileLocation = self.daofactory(
classname="Files.SetLocationByLFN")
self.setFileAddChecksum = self.daofactory(
classname="Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname="Files.Add")
self.jobCompleteInput = self.daofactory(classname="Jobs.CompleteInput")
self.setBulkOutcome = self.daofactory(classname="Jobs.SetOutcomeBulk")
self.getWorkflowSpec = self.daofactory(
classname="Workflow.GetSpecAndNameFromTask")
self.getJobInfoByID = self.daofactory(classname="Jobs.LoadFromID")
self.getFullJobInfo = self.daofactory(
classname="Jobs.LoadForErrorHandler")
self.getJobTaskNameAction = self.daofactory(
classname="Jobs.GetFWJRTaskName")
self.pnn_to_psn = self.daofactory(
classname="Locations.GetPNNtoPSNMapping").execute()
self.dbsStatusAction = self.dbsDaoFactory(
classname="DBSBufferFiles.SetStatus")
self.dbsParentStatusAction = self.dbsDaoFactory(
classname="DBSBufferFiles.GetParentStatus")
self.dbsChildrenAction = self.dbsDaoFactory(
classname="DBSBufferFiles.GetChildren")
self.dbsCreateFiles = self.dbsDaoFactory(
classname="DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(
classname="DBSBufferFiles.AddChecksumByLFN")
self.dbsSetRunLumi = self.dbsDaoFactory(
classname="DBSBufferFiles.AddRunLumi")
self.dbsGetWorkflow = self.dbsDaoFactory(classname="ListWorkflow")
self.dbsLFNHeritage = self.dbsDaoFactory(
classname="DBSBufferFiles.BulkHeritageParent")
self.stateChanger = ChangeState(config)
# Decide whether or not to attach jobReport to returned value
self.returnJobReport = getattr(config.JobAccountant,
'returnReportFromWorker', False)
# Store location for the specs for DBS
self.specDir = getattr(config.JobAccountant, 'specDir', None)
# maximum RAW EDM size for Repack output before data is put into Error dataset and skips PromptReco
self.maxAllowedRepackOutputSize = getattr(
config.JobAccountant, 'maxAllowedRepackOutputSize',
12 * 1024 * 1024 * 1024)
# ACDC service
self.dataCollection = DataCollectionService(
url=config.ACDC.couchurl, database=config.ACDC.database)
jobDBurl = sanitizeURL(config.JobStateMachine.couchurl)['url']
jobDBName = config.JobStateMachine.couchDBName
jobCouchdb = CouchServer(jobDBurl)
self.fwjrCouchDB = jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
self.localWMStats = WMStatsWriter(config.TaskArchiver.localWMStatsURL,
appName="WMStatsAgent")
# Hold data for later commital
self.dbsFilesToCreate = []
self.wmbsFilesToBuild = []
self.wmbsMergeFilesToBuild = []
self.fileLocation = None
self.mergedOutputFiles = []
self.listOfJobsToSave = []
self.listOfJobsToFail = []
self.filesetAssoc = []
self.parentageBinds = []
self.parentageBindsForMerge = []
self.jobsWithSkippedFiles = {}
self.count = 0
self.datasetAlgoID = collections.deque(maxlen=1000)
self.datasetAlgoPaths = collections.deque(maxlen=1000)
self.dbsLocations = set()
self.workflowIDs = collections.deque(maxlen=1000)
self.workflowPaths = collections.deque(maxlen=1000)
self.phedex = PhEDEx()
self.locLists = self.phedex.getNodeMap()
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
Split files into a number of lumis per job
Allow a flag to determine if we split files between jobs
"""
avgEventsPerJob = int(kwargs.get('events_per_job', 5000))
jobLimit = int(kwargs.get('job_limit', 0))
jobTimeLimit = int(
kwargs.get('job_time_limit', self.defaultJobTimeLimit))
totalEvents = int(kwargs.get('total_events', 0))
splitOnFile = bool(kwargs.get('halt_job_on_file_boundaries', False))
self.collectionName = kwargs.get('collectionName', None)
splitOnRun = kwargs.get('splitOnRun', True)
getParents = kwargs.get('include_parents', False)
runWhitelist = kwargs.get('runWhitelist', [])
runs = kwargs.get('runs', None)
lumis = kwargs.get('lumis', None)
applyLumiCorrection = bool(kwargs.get('applyLumiCorrection', False))
deterministicPileup = kwargs.get('deterministicPileup', False)
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
eventsPerLumiInDataset = 0
if avgEventsPerJob <= 0:
msg = "events_per_job parameter must be positive. Its value is: %d" % avgEventsPerJob
raise RuntimeError(msg)
if self.package == 'WMCore.WMBS':
self.loadRunLumi = self.daoFactory(
classname="Files.GetBulkRunLumi")
if deterministicPileup:
getJobNumber = self.daoFactory(
classname="Jobs.GetNumberOfJobsPerWorkflow")
self.nJobs = getJobNumber.execute(
workflow=self.subscription.getWorkflow().id)
logging.info(
'Creating jobs in DeterministicPileup mode for %s',
self.subscription.workflowName())
goodRunList = {}
if runs and lumis:
goodRunList = buildLumiMask(runs, lumis)
# If we have runLumi info, we need to load it from couch
if self.collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
logging.info('Creating jobs for ACDC fileset %s', filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumiWhitelist(self.collectionName,
filesetName)
except Exception as ex:
msg = "Exception while trying to load goodRunList. "
msg += "Refusing to create any jobs.\nDetails: %s" % str(ex)
logging.exception(msg)
return
lDict = self.getFilesSortedByLocation(avgEventsPerJob)
if not lDict:
logging.info(
"There are not enough events/files to be splitted. Trying again next cycle"
)
return
locationDict = {}
for key in lDict.keys():
newlist = []
# First we need to load the data
if self.loadRunLumi:
fileLumis = self.loadRunLumi.execute(files=lDict[key])
if not fileLumis:
logging.warning(
"Empty fileLumis dict for workflow %s, subs %s.",
self.subscription.workflowName(),
self.subscription['id'])
for f in lDict[key]:
lumiDict = fileLumis.get(f['id'], {})
for run in lumiDict.keys():
f.addRun(run=Run(run, *lumiDict[run]))
for f in lDict[key]:
if len(f['runs']) == 0:
continue
f['runs'] = sorted(f['runs'])
f['lumiCount'] = 0
for run in f['runs']:
run.lumis.sort()
f['lumiCount'] += len(run.lumis)
f['lowestRun'] = f['runs'][0]
# Do average event per lumi calculation
if f['lumiCount']:
f['avgEvtsPerLumi'] = round(
float(f['events']) / f['lumiCount'])
if deterministicPileup:
# We assume that all lumis are equal in the dataset
eventsPerLumiInDataset = f['avgEvtsPerLumi']
else:
# No lumis in the file, ignore it
continue
newlist.append(f)
locationDict[key] = sorted(newlist,
key=operator.itemgetter('lowestRun'))
totalJobs = 0
lastLumi = None
firstLumi = None
lastRun = None
lumisInJob = 0
totalAvgEventCount = 0
currentJobAvgEventCount = 0
stopTask = False
self.lumiChecker = LumiChecker(applyLumiCorrection)
for location in locationDict:
# For each location, we need a new jobGroup
self.newGroup()
stopJob = True
for f in locationDict[location]:
if getParents:
parentLFNs = self.findParent(lfn=f['lfn'])
for lfn in parentLFNs:
parent = File(lfn=lfn)
f['parents'].add(parent)
lumisInJobInFile = 0
updateSplitOnJobStop = False
failNextJob = False
# If estimated job time is higher the job time limit (condor limit)
# and it's only one lumi then ditch that lumi
timePerLumi = f['avgEvtsPerLumi'] * timePerEvent
if timePerLumi > jobTimeLimit and f['lumiCount'] == 1:
failNextJob = True
stopJob = True
lumisPerJob = 1
elif splitOnFile:
# Then we have to split on every boundary
stopJob = True
# Check the average number of events per lumi in this file
# Adapt the lumis per job to match the target conditions
if f['avgEvtsPerLumi']:
# If there are events in the file
ratio = float(avgEventsPerJob) / f['avgEvtsPerLumi']
lumisPerJob = max(int(math.floor(ratio)), 1)
else:
# Zero event file, then the ratio goes to infinity. Computers don't like that
lumisPerJob = f['lumiCount']
else:
# Analyze how many events does this job already has
# Check how many we want as target, include as many lumi sections as possible
updateSplitOnJobStop = True
eventsRemaining = max(
avgEventsPerJob - currentJobAvgEventCount, 0)
if f['avgEvtsPerLumi']:
lumisAllowed = int(
math.floor(
float(eventsRemaining) / f['avgEvtsPerLumi']))
else:
lumisAllowed = f['lumiCount']
lumisPerJob = max(lumisInJob + lumisAllowed, 1)
for run in f['runs']:
if not isGoodRun(goodRunList=goodRunList, run=run.run):
# Then skip this one
continue
if len(runWhitelist) > 0 and not run.run in runWhitelist:
# Skip due to run whitelist
continue
firstLumi = None
if splitOnRun and run.run != lastRun:
# Then we need to kill this job and get a new one
stopJob = True
# Now loop over the lumis
for lumi in run:
if (not isGoodLumi(goodRunList, run=run.run, lumi=lumi)
or self.lumiChecker.isSplitLumi(
run.run, lumi, f)):
# Kill the chain of good lumis
# Skip this lumi
if firstLumi != None and firstLumi != lumi:
self.currentJob['mask'].addRunAndLumis(
run=run.run, lumis=[firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
firstLumi = None
lastLumi = None
continue
# You have to kill the lumi chain if they're not continuous
if lastLumi and not lumi == lastLumi + 1:
self.currentJob['mask'].addRunAndLumis(
run=run.run, lumis=[firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
firstLumi = None
lastLumi = None
if firstLumi is None:
# Set the first lumi in the run
firstLumi = lumi
# If we're full, end the job
if lumisInJob == lumisPerJob:
stopJob = True
# Actually do the new job creation
if stopJob:
if firstLumi != None and lastLumi != None and lastRun != None:
self.currentJob['mask'].addRunAndLumis(
run=lastRun, lumis=[firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
msg = None
if failNextJob:
msg = "File %s has a single lumi %s, in run %s " % (
f['lfn'], lumi, run.run)
msg += "with too many events %d and it woud take %d sec to run" \
% (f['events'], timePerLumi)
self.lumiChecker.closeJob(self.currentJob)
self.newJob(name=self.getJobName(),
failedJob=failNextJob,
failedReason=msg)
if deterministicPileup:
skipEvents = (
self.nJobs -
1) * lumisPerJob * eventsPerLumiInDataset
self.currentJob.addBaggageParameter(
"skipPileupEvents", skipEvents)
self.currentJob.addResourceEstimates(
memory=memoryRequirement)
failNextJob = False
firstLumi = lumi
lumisInJob = 0
lumisInJobInFile = 0
currentJobAvgEventCount = 0
totalJobs += 1
if jobLimit and totalJobs > jobLimit:
msg = "Job limit of {0} jobs exceeded.".format(
jobLimit)
raise RuntimeError(msg)
# Add the file to new jobs
self.currentJob.addFile(f)
if updateSplitOnJobStop:
# Then we were carrying from a previous file
# Reset calculations for this file
updateSplitOnJobStop = False
if f['avgEvtsPerLumi']:
ratio = float(
avgEventsPerJob) / f['avgEvtsPerLumi']
lumisPerJob = max(int(math.floor(ratio)),
1)
else:
lumisPerJob = f['lumiCount']
lumisInJob += 1
lumisInJobInFile += 1
lastLumi = lumi
stopJob = False
lastRun = run.run
totalAvgEventCount += f['avgEvtsPerLumi']
if self.currentJob and not f in self.currentJob[
'input_files']:
self.currentJob.addFile(f)
# We stop here if there are more total events than requested.
if totalEvents > 0 and totalAvgEventCount >= totalEvents:
stopTask = True
break
if firstLumi != None and lastLumi != None:
# Add this run to the mask
self.currentJob['mask'].addRunAndLumis(
run=run.run, lumis=[firstLumi, lastLumi])
eventsAdded = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = eventsAdded * timePerEvent
runAddedSize = eventsAdded * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
firstLumi = None
lastLumi = None
if stopTask:
break
if not splitOnFile:
currentJobAvgEventCount += f[
'avgEvtsPerLumi'] * lumisInJobInFile
if stopTask:
break
if stopTask:
break
self.lumiChecker.closeJob(self.currentJob)
self.lumiChecker.fixInputFiles()
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
An event base splitting algorithm. All available files are split into a
set number of events per job.
"""
eventsPerJob = int(kwargs.get("events_per_job", 100))
eventsPerLumi = int(kwargs.get("events_per_lumi", eventsPerJob))
getParents = kwargs.get("include_parents", False)
lheInput = kwargs.get("lheInputFiles", False)
collectionName = kwargs.get('collectionName', None)
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
acdcFileList = []
deterministicPileup = kwargs.get('deterministicPileup', False)
if eventsPerJob <= 0 or eventsPerLumi <= 0:
msg = "events_per_job and events_per_lumi must be positive. Their values are: "
msg += "events_per_job: %d, events_per_lumi: %d" % (eventsPerJob, eventsPerLumi)
raise RuntimeError(msg)
if deterministicPileup and self.package == 'WMCore.WMBS':
getJobNumber = self.daoFactory(classname="Jobs.GetNumberOfJobsPerWorkflow")
self.nJobs = getJobNumber.execute(workflow=self.subscription.getWorkflow().id)
logging.info('Creating jobs in DeterministicPileup mode for %s',
self.subscription.workflowName())
# If we have runLumi info, we need to load it from couch
if collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
collectionName = kwargs.get('collectionName')
logging.info('Loading ACDC info for collectionName: %s, with filesetName: %s', collectionName,
filesetName)
dcs = DataCollectionService(couchURL, couchDB)
acdcFileList = dcs.getProductionACDCInfo(collectionName, filesetName)
except Exception as ex:
msg = "Exception while trying to load goodRunList\n"
msg += "Refusing to create any jobs.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
return
totalJobs = 0
locationDict = self.sortByLocation()
for location in locationDict:
self.newGroup()
fileList = locationDict[location]
getRunLumiInformation = False
for f in fileList:
if f['lfn'].startswith("MCFakeFile"):
# We have one MCFakeFile, then it needs run information
getRunLumiInformation = True
break
if getRunLumiInformation:
if self.package == 'WMCore.WMBS':
loadRunLumi = self.daoFactory(classname="Files.GetBulkRunLumi")
fileLumis = loadRunLumi.execute(files=fileList)
if not fileLumis:
logging.warning("Empty fileLumis dict for workflow %s, subs %s.",
self.subscription.workflowName(), self.subscription['id'])
for f in fileList:
lumiDict = fileLumis.get(f['id'], {})
for run in lumiDict:
f.addRun(run=Run(run, *lumiDict[run]))
for f in fileList:
currentEvent = f['first_event']
eventsInFile = f['events']
runs = list(f['runs'])
# We got the runs, clean the file.
f['runs'] = set()
if getParents:
parentLFNs = self.findParent(lfn=f['lfn'])
for lfn in parentLFNs:
parent = File(lfn=lfn)
f['parents'].add(parent)
if acdcFileList:
totalJobs = self.createACDCJobs(f, acdcFileList, timePerEvent,
sizePerEvent, memoryRequirement,
lheInput, eventsPerJob, eventsPerLumi,
deterministicPileup, totalJobs)
continue
if not f['lfn'].startswith("MCFakeFile"):
# there might be files with 0 event that still have to be processed
if eventsInFile == 0:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
# Do not set LastEvent
self.currentJob["mask"].setMaxAndSkipEvents(None, currentEvent)
self.currentJob.addResourceEstimates(jobTime=0,
memory=memoryRequirement,
disk=0)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
totalJobs += 1
logging.info("Job created for 0-event input file with %s", self.currentJob)
# Very very uncommon in production, but it has real input dataset
while eventsInFile:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
if eventsInFile >= eventsPerJob:
jobTime = eventsPerJob * timePerEvent
diskRequired = eventsPerJob * sizePerEvent
self.currentJob["mask"].setMaxAndSkipEvents(eventsPerJob - 1, currentEvent)
else:
jobTime = eventsInFile * timePerEvent
diskRequired = eventsInFile * sizePerEvent
self.currentJob["mask"].setMaxAndSkipEvents(eventsInFile - 1, currentEvent)
eventsInFile = eventsPerJob
self.currentJob.addResourceEstimates(jobTime=jobTime,
memory=memoryRequirement,
disk=diskRequired)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
eventsInFile -= eventsPerJob
currentEvent += eventsPerJob
totalJobs += 1
logging.debug("Job created for real input with %s", self.currentJob)
else:
# This assumes there's only one run which is the case for MC
lumis = runs[0].lumis
(firstLumi, lastLumi) = (min(lumis), max(lumis))
currentLumi = firstLumi
lumisPerJob = int(ceil(float(eventsPerJob) / eventsPerLumi))
while eventsInFile:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
self.currentJob.addBaggageParameter("lheInputFiles", lheInput)
# Limit the number of events to a unsigned 32bit int
if (currentEvent + eventsPerJob - 1) > (2 ** 32 - 1) and \
(currentEvent + eventsInFile) > (2 ** 32 - 1):
currentEvent = 1
if eventsInFile >= eventsPerJob:
jobTime = eventsPerJob * timePerEvent
diskRequired = eventsPerJob * sizePerEvent
# Alan on 16/Apr/2019: inclusiveMask must be a real inclusiveMask, thus
# FirstEvent/FirstLumi and LastEvent/LastLumi are also processed by the job
self.currentJob["mask"].setMaxAndSkipEvents(eventsPerJob - 1, currentEvent)
self.currentJob["mask"].setMaxAndSkipLumis(lumisPerJob - 1, currentLumi)
else:
jobTime = eventsInFile * timePerEvent
diskRequired = eventsInFile * sizePerEvent
lumisPerJob = int(ceil(float(eventsInFile) / eventsPerLumi))
self.currentJob["mask"].setMaxAndSkipEvents(eventsInFile - 1, currentEvent)
self.currentJob["mask"].setMaxAndSkipLumis(lumisPerJob - 1, currentLumi)
eventsInFile = eventsPerJob
self.currentJob.addResourceEstimates(jobTime=jobTime,
memory=memoryRequirement,
disk=diskRequired)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
eventsInFile -= eventsPerJob
currentEvent += eventsPerJob
currentLumi += lumisPerJob
totalJobs += 1
logging.info("Job created with mask: %s", self.currentJob['mask'])
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
Split files into a number of lumis per job
Allow a flag to determine if we split files between jobs
"""
lumisPerJob = int(kwargs.get('lumis_per_job', 1))
totalLumis = int(kwargs.get('total_lumis', 0))
splitOnFile = bool(kwargs.get('halt_job_on_file_boundaries', False))
self.collectionName = kwargs.get('collectionName', None)
splitOnRun = kwargs.get('splitOnRun', True)
getParents = kwargs.get('include_parents', False)
runWhitelist = kwargs.get('runWhitelist', [])
runs = kwargs.get('runs', None)
lumis = kwargs.get('lumis', None)
deterministicPileup = kwargs.get('deterministicPileup', False)
applyLumiCorrection = bool(kwargs.get('applyLumiCorrection', False))
eventsPerLumiInDataset = 0
if self.package == 'WMCore.WMBS':
self.loadRunLumi = self.daoFactory(
classname="Files.GetBulkRunLumi")
if deterministicPileup:
getJobNumber = self.daoFactory(
classname="Jobs.GetNumberOfJobsPerWorkflow")
self.nJobs = getJobNumber.execute(
workflow=self.subscription.getWorkflow().id)
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
goodRunList = {}
if runs and lumis:
goodRunList = buildLumiMask(runs, lumis)
# If we have runLumi info, we need to load it from couch
if self.collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
logging.info('Creating jobs for ACDC fileset %s', filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumiWhitelist(self.collectionName,
filesetName)
except Exception as ex:
msg = "Exception while trying to load goodRunList. "
msg += "Refusing to create any jobs.\nDetails: %s" % str(ex)
logging.exception(msg)
return
lDict = self.getFilesSortedByLocation(lumisPerJob)
if not lDict:
logging.info(
"There are not enough lumis/files to be splitted. Trying again next cycle"
)
return
locationDict = {}
for key in lDict.keys():
newlist = []
for f in lDict[key]:
# if hasattr(f, 'loadData'):
# f.loadData()
if len(f['runs']) == 0:
continue
f['lumiCount'] = 0
f['runs'] = sorted(f['runs'])
for run in f['runs']:
run.lumis.sort()
f['lumiCount'] += len(run.lumis)
f['lowestRun'] = f['runs'][0]
# Do average event per lumi calculation
if f['lumiCount']:
f['avgEvtsPerLumi'] = round(
float(f['events']) / f['lumiCount'])
if deterministicPileup:
# We assume that all lumis are equal in the dataset
eventsPerLumiInDataset = f['avgEvtsPerLumi']
else:
# No lumis in the file, ignore it
continue
newlist.append(f)
locationDict[key] = sorted(newlist,
key=operator.itemgetter('lowestRun'))
# Split files into jobs with each job containing
# EXACTLY lumisPerJob number of lumis (except for maybe the last one)
totalJobs = 0
lastLumi = None
firstLumi = None
stopJob = True
stopTask = False
lastRun = None
lumisInJob = 0
lumisInTask = 0
self.lumiChecker = LumiChecker(applyLumiCorrection)
for location in locationDict.keys():
# For each location, we need a new jobGroup
self.newGroup()
stopJob = True
for f in locationDict[location]:
if getParents:
parentLFNs = self.findParent(lfn=f['lfn'])
for lfn in parentLFNs:
parent = File(lfn=lfn)
f['parents'].add(parent)
if splitOnFile:
# Then we have to split on every boundary
stopJob = True
for run in f['runs']:
if not isGoodRun(goodRunList=goodRunList, run=run.run):
# Then skip this one
continue
if len(runWhitelist) > 0 and not run.run in runWhitelist:
# Skip due to run whitelist
continue
firstLumi = None
if splitOnRun and run.run != lastRun:
# Then we need to kill this job and get a new one
stopJob = True
# Now loop over the lumis
for lumi in run:
# splitLumi checks if the lumi is split across jobs
if (not isGoodLumi(goodRunList, run=run.run, lumi=lumi)
or self.lumiChecker.isSplitLumi(
run.run, lumi, f)):
# Kill the chain of good lumis
# Skip this lumi
if firstLumi != None and firstLumi != lumi:
self.currentJob['mask'].addRunAndLumis(
run=run.run, lumis=[firstLumi, lastLumi])
addedEvents = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = addedEvents * timePerEvent
runAddedSize = addedEvents * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
firstLumi = None
lastLumi = None
continue
# You have to kill the lumi chain if they're not continuous
if lastLumi and not lumi == lastLumi + 1:
self.currentJob['mask'].addRunAndLumis(
run=run.run, lumis=[firstLumi, lastLumi])
addedEvents = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = addedEvents * timePerEvent
runAddedSize = addedEvents * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
firstLumi = None
lastLumi = None
if firstLumi is None:
# Set the first lumi in the run
firstLumi = lumi
# If we're full, end the job
if lumisInJob == lumisPerJob:
stopJob = True
# Actually do the new job creation
if stopJob:
if firstLumi != None and lastLumi != None and lastRun != None:
self.currentJob['mask'].addRunAndLumis(
run=lastRun, lumis=[firstLumi, lastLumi])
addedEvents = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = addedEvents * timePerEvent
runAddedSize = addedEvents * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
# before creating a new job add the lumis of the current one to the checker
self.lumiChecker.closeJob(self.currentJob)
self.newJob(name=self.getJobName())
self.currentJob.addResourceEstimates(
memory=memoryRequirement)
if deterministicPileup:
skipEvents = (
self.nJobs -
1) * lumisPerJob * eventsPerLumiInDataset
self.currentJob.addBaggageParameter(
"skipPileupEvents", skipEvents)
firstLumi = lumi
lumisInJob = 0
totalJobs += 1
# Add the file to new jobs
self.currentJob.addFile(f)
lumisInJob += 1
lumisInTask += 1
lastLumi = lumi
stopJob = False
lastRun = run.run
if self.currentJob and not f in self.currentJob[
'input_files']:
self.currentJob.addFile(f)
if totalLumis > 0 and lumisInTask >= totalLumis:
stopTask = True
break
if firstLumi != None and lastLumi != None:
# Add this run to the mask
self.currentJob['mask'].addRunAndLumis(
run=run.run, lumis=[firstLumi, lastLumi])
addedEvents = ((lastLumi - firstLumi + 1) *
f['avgEvtsPerLumi'])
runAddedTime = addedEvents * timePerEvent
runAddedSize = addedEvents * sizePerEvent
self.currentJob.addResourceEstimates(
jobTime=runAddedTime, disk=runAddedSize)
firstLumi = None
lastLumi = None
if stopTask:
break
if stopTask:
break
if stopTask:
break
self.lumiChecker.closeJob(self.currentJob)
self.lumiChecker.fixInputFiles()
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
Split files into a number of lumis per job
Allow a flag to determine if we split files between jobs
"""
myThread = threading.currentThread()
lumisPerJob = int(kwargs.get('lumis_per_job', 1))
splitOnFile = bool(kwargs.get('halt_job_on_file_boundaries', True))
ignoreACDC = bool(kwargs.get('ignore_acdc_except', False))
collectionName = kwargs.get('collectionName', None)
splitOnRun = kwargs.get('splitOnRun', True)
getParents = kwargs.get('include_parents', False)
runWhitelist = kwargs.get('runWhitelist', [])
runs = kwargs.get('runs', None)
lumis = kwargs.get('lumis', None)
deterministicPileup = kwargs.get('deterministicPileup', False)
eventsPerLumiInDataset = 0
if deterministicPileup and self.package == 'WMCore.WMBS':
getJobNumber = self.daoFactory(
classname="Jobs.GetNumberOfJobsPerWorkflow")
jobNumber = getJobNumber.execute(
workflow=self.subscription.getWorkflow().id)
self.nJobs = jobNumber
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
goodRunList = {}
if runs and lumis:
goodRunList = buildLumiMask(runs, lumis)
# If we have runLumi info, we need to load it from couch
if collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
collectionName = kwargs.get('collectionName')
owner = kwargs.get('owner')
group = kwargs.get('group')
logging.info('Creating jobs for ACDC fileset %s' % filesetName)
dcs = DataCollectionService(couchURL, couchDB)
goodRunList = dcs.getLumiWhitelist(collectionName, filesetName,
owner, group)
except Exception, ex:
msg = "Exception while trying to load goodRunList\n"
if ignoreACDC:
msg += "Ditching goodRunList\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
goodRunList = {}
else:
msg += "Refusing to create any jobs.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
return
def algorithm(self, *args, **kwargs):
"""
_algorithm_
An event base splitting algorithm. All available files are split into a
set number of events per job.
"""
eventsPerJob = int(kwargs.get("events_per_job", 100))
eventsPerLumi = int(kwargs.get("events_per_lumi", eventsPerJob))
getParents = kwargs.get("include_parents", False)
lheInput = kwargs.get("lheInputFiles", False)
collectionName = kwargs.get('collectionName', None)
timePerEvent, sizePerEvent, memoryRequirement = \
self.getPerformanceParameters(kwargs.get('performance', {}))
acdcFileList = []
deterministicPileup = kwargs.get('deterministicPileup', False)
if deterministicPileup and self.package == 'WMCore.WMBS':
getJobNumber = self.daoFactory(classname="Jobs.GetNumberOfJobsPerWorkflow")
self.nJobs = getJobNumber.execute(workflow=self.subscription.getWorkflow().id)
logging.info('Creating %d jobs in DeterministicPileup mode', self.nJobs)
# If we have runLumi info, we need to load it from couch
if collectionName:
try:
from WMCore.ACDC.DataCollectionService import DataCollectionService
couchURL = kwargs.get('couchURL')
couchDB = kwargs.get('couchDB')
filesetName = kwargs.get('filesetName')
collectionName = kwargs.get('collectionName')
owner = kwargs.get('owner')
group = kwargs.get('group')
logging.info('Creating jobs for ACDC fileset %s', filesetName)
dcs = DataCollectionService(couchURL, couchDB)
acdcFileList = dcs.getProductionACDCInfo(collectionName, filesetName, owner, group)
except Exception as ex:
msg = "Exception while trying to load goodRunList\n"
msg += "Refusing to create any jobs.\n"
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
return
totalJobs = 0
locationDict = self.sortByLocation()
for location in locationDict:
self.newGroup()
fileList = locationDict[location]
getRunLumiInformation = False
for f in fileList:
if f['lfn'].startswith("MCFakeFile"):
# We have one MCFakeFile, then it needs run information
getRunLumiInformation = True
break
if getRunLumiInformation:
if self.package == 'WMCore.WMBS':
loadRunLumi = self.daoFactory(classname="Files.GetBulkRunLumi")
fileLumis = loadRunLumi.execute(files=fileList)
for f in fileList:
lumiDict = fileLumis.get(f['id'], {})
for run in lumiDict.keys():
f.addRun(run=Run(run, *lumiDict[run]))
for f in fileList:
currentEvent = f['first_event']
eventsInFile = f['events']
runs = list(f['runs'])
# We got the runs, clean the file.
f['runs'] = set()
if getParents:
parentLFNs = self.findParent(lfn=f['lfn'])
for lfn in parentLFNs:
parent = File(lfn=lfn)
f['parents'].add(parent)
if acdcFileList:
if f['lfn'] in [x['lfn'] for x in acdcFileList]:
totalJobs = self.createACDCJobs(f, acdcFileList, timePerEvent,
sizePerEvent, memoryRequirement,
lheInput, eventsPerJob, eventsPerLumi,
deterministicPileup, totalJobs)
continue
if not f['lfn'].startswith("MCFakeFile"):
# Very very uncommon, but it has real input dataset
if eventsInFile >= eventsPerJob:
while currentEvent < eventsInFile:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
if eventsPerJob + currentEvent < eventsInFile:
jobTime = eventsPerJob * timePerEvent
diskRequired = eventsPerJob * sizePerEvent
self.currentJob["mask"].setMaxAndSkipEvents(eventsPerJob, currentEvent)
else:
jobTime = (eventsInFile - currentEvent) * timePerEvent
diskRequired = (eventsInFile - currentEvent) * sizePerEvent
self.currentJob["mask"].setMaxAndSkipEvents(None,
currentEvent)
self.currentJob.addResourceEstimates(jobTime=jobTime,
memory=memoryRequirement,
disk=diskRequired)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
logging.debug("Job created for real input with %s", self.currentJob)
currentEvent += eventsPerJob
totalJobs += 1
else:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
jobTime = eventsInFile * timePerEvent
diskRequired = eventsInFile * sizePerEvent
self.currentJob.addResourceEstimates(jobTime=jobTime,
memory=memoryRequirement,
disk=diskRequired)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
logging.debug("Last job created for real input with %s", self.currentJob)
totalJobs += 1
else:
# This assumes there's only one run which is the case for MC
lumis = runs[0].lumis
(firstLumi, lastLumi) = (min(lumis), max(lumis))
currentLumi = firstLumi
totalEvents = 0
if eventsInFile >= eventsPerJob:
while totalEvents < eventsInFile:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
self.currentJob.addBaggageParameter("lheInputFiles", lheInput)
lumisPerJob = int(ceil(float(eventsPerJob)
/ eventsPerLumi))
# Limit the number of events to a unsigned 32bit int
eventsRemaining = eventsInFile - totalEvents
if (currentEvent + eventsPerJob - 1) > (2 ** 32 - 1) and (
currentEvent + eventsRemaining - 1) > (2 ** 32 - 1):
currentEvent = 1
if eventsRemaining > eventsPerJob:
self.currentJob["mask"].setMaxAndSkipEvents(eventsPerJob,
currentEvent)
self.currentJob["mask"].setMaxAndSkipLumis(lumisPerJob,
currentLumi)
jobTime = eventsPerJob * timePerEvent
diskRequired = eventsPerJob * sizePerEvent
else:
jobTime = eventsRemaining * timePerEvent
diskRequired = eventsRemaining * sizePerEvent
lumisPerJob = int(ceil(float(eventsRemaining) / eventsPerLumi))
self.currentJob["mask"].setMaxAndSkipEvents(eventsRemaining,
currentEvent)
self.currentJob["mask"].setMaxAndSkipLumis(lumisPerJob,
currentLumi)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
currentLumi += lumisPerJob
currentEvent += eventsPerJob
totalEvents += eventsPerJob
totalJobs += 1
self.currentJob.addResourceEstimates(jobTime=jobTime,
memory=memoryRequirement,
disk=diskRequired)
else:
self.newJob(name=self.getJobName(length=totalJobs))
self.currentJob.addFile(f)
# For MC we use firstEvent instead of skipEvents so set it to 1
# We must check for events going over 2**32 - 1 here too
if (eventsInFile + currentEvent - 1) > (2 ** 32 - 1):
currentEvent = 1
self.currentJob["mask"].setMaxAndSkipEvents(eventsInFile,
currentEvent)
self.currentJob["mask"].setMaxAndSkipLumis(lastLumi -
currentLumi + 1, currentLumi)
jobTime = eventsInFile * timePerEvent
diskRequired = eventsInFile * sizePerEvent
self.currentJob.addResourceEstimates(jobTime=jobTime,
memory=memoryRequirement,
disk=diskRequired)
if deterministicPileup:
self.currentJob.addBaggageParameter("skipPileupEvents", (self.nJobs - 1) * eventsPerJob)
totalJobs += 1
def testChunking(self):
"""
_testChunking_
Insert a workload and files that have several distinct sets of
locations. Verify that the chunks are created correctly and that they
only groups files that have the same set of locations. Also verify that
the chunks are pulled out of ACDC correctly.
"""
dcs = DataCollectionService(url=self.testInit.couchUrl,
database="wmcore-acdc-datacollectionsvc")
testFileA = File(lfn=makeUUID(), size=1024, events=1024)
testFileA.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"])
testFileA.addRun(Run(1, 1, 2))
testFileB = File(lfn=makeUUID(), size=1024, events=1024)
testFileB.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"])
testFileB.addRun(Run(1, 3, 4))
testFileC = File(lfn=makeUUID(), size=1024, events=1024)
testFileC.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"])
testFileC.addRun(Run(1, 5, 6))
testJobA = self.getMinimalJob()
testJobA.addFile(testFileA)
testJobA.addFile(testFileB)
testJobA.addFile(testFileC)
testFileD = File(lfn=makeUUID(), size=1024, events=1024)
testFileD.setLocation(["cmssrm.fnal.gov"])
testFileD.addRun(Run(2, 1, 2))
testFileE = File(lfn=makeUUID(), size=1024, events=1024)
testFileE.setLocation(["cmssrm.fnal.gov"])
testFileE.addRun(Run(2, 3, 4))
testJobB = self.getMinimalJob()
testJobB.addFile(testFileD)
testJobB.addFile(testFileE)
testFileF = File(lfn=makeUUID(),
size=1024,
events=1024,
parents={"/some/parent/F"})
testFileF.setLocation(
["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"])
testFileF.addRun(Run(3, 1, 2))
testFileG = File(lfn=makeUUID(),
size=1024,
events=1024,
parents={"/some/parent/G"})
testFileG.setLocation(
["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"])
testFileG.addRun(Run(3, 3, 4))
testFileH = File(lfn=makeUUID(),
size=1024,
events=1024,
parents={"/some/parent/H"})
testFileH.setLocation(
["cmssrm.fnal.gov", "castor.cern.ch", "srm.ral.uk"])
testFileH.addRun(Run(3, 5, 6))
testJobC = self.getMinimalJob()
testJobC.addFile(testFileF)
testJobC.addFile(testFileG)
testJobC.addFile(testFileH)
testFileI = File(lfn=makeUUID(), size=1024, events=1024, merged=True)
testFileI.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"])
testFileI.addRun(Run(4, 1, 2))
testFileJ = File(lfn=makeUUID(), size=1024, events=1024, merged=True)
testFileJ.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"])
testFileJ.addRun(Run(4, 3, 4))
testFileK = File(lfn=makeUUID(), size=1024, events=1024, merged=True)
testFileK.setLocation(["cmssrm.fnal.gov", "castor.cern.ch"])
testFileK.addRun(Run(4, 5, 6))
testJobD = self.getMinimalJob()
testJobD.addFile(testFileI)
testJobD.addFile(testFileJ)
testJobD.addFile(testFileK)
dcs.failedJobs([testJobA, testJobB, testJobC, testJobD])
chunks = dcs.chunkFileset("ACDCTest", "/ACDCTest/reco", chunkSize=5)
self.assertEqual(
len(chunks), 4,
"Error: There should be four chunks: %s" % len(chunks))
goldenMetaData = {
1: {
"lumis": 2,
"locations": ["castor.cern.ch", "cmssrm.fnal.gov"],
"events": 1024
},
2: {
"lumis": 4,
"locations": ["cmssrm.fnal.gov"],
"events": 2048
},
3: {
"lumis": 6,
"locations":
["castor.cern.ch", "cmssrm.fnal.gov", "srm.ral.uk"],
"events": 3072
},
5: {
"lumis": 10,
"locations": ["castor.cern.ch", "cmssrm.fnal.gov"],
"events": 5120
}
}
testFiles = [
testFileA, testFileB, testFileC, testFileI, testFileJ, testFileK
]
lastFile = testFileA
for testFile in testFiles:
if lastFile["lfn"] < testFile["lfn"]:
lastFile = testFile
testFiles.remove(lastFile)
goldenFiles = {
1: [lastFile],
2: [testFileD, testFileE],
3: [testFileF, testFileG, testFileH],
5: testFiles
}
for chunk in chunks:
chunkMetaData = dcs.getChunkInfo("ACDCTest", "/ACDCTest/reco",
chunk["offset"], chunk["files"])
self.assertEqual(chunkMetaData["files"], chunk["files"])
self.assertEqual(chunkMetaData["lumis"], chunk["lumis"])
self.assertEqual(chunkMetaData["events"], chunk["events"])
self.assertEqual(chunkMetaData["locations"], chunk["locations"])
self.assertTrue(chunk["files"] in goldenMetaData.keys(),
"Error: Extra chunk found.")
self.assertEqual(chunk["lumis"],
goldenMetaData[chunk["files"]]["lumis"],
"Error: Lumis in chunk is wrong.")
self.assertEqual(chunk["locations"],
goldenMetaData[chunk["files"]]["locations"],
"Error: Locations in chunk is wrong.")
self.assertEqual(chunk["events"],
goldenMetaData[chunk["files"]]["events"],
"Error: Events in chunk is wrong.")
del goldenMetaData[chunk["files"]]
chunkFiles = dcs.getChunkFiles("ACDCTest", "/ACDCTest/reco",
chunk["offset"], chunk["files"])
self.assertTrue(chunk["files"] in goldenFiles.keys(),
"Error: Extra chunk found.")
goldenChunkFiles = goldenFiles[chunk["files"]]
self.assertEqual(len(chunkFiles), len(goldenChunkFiles))
for chunkFile in chunkFiles:
foundFile = None
for goldenChunkFile in goldenChunkFiles:
if chunkFile["lfn"] == goldenChunkFile["lfn"]:
foundFile = goldenChunkFile
break
self.assertIsNotNone(
foundFile, "Error: Missing chunk file: %s, %s" %
(chunkFiles, goldenChunkFiles))
self.assertEqual(set(foundFile["parents"]),
chunkFile["parents"],
"Error: File parents should match.")
self.assertEqual(foundFile["merged"], chunkFile["merged"],
"Error: File merged status should match.")
self.assertEqual(foundFile["locations"],
chunkFile["locations"],
"Error: File locations should match.")
self.assertEqual(foundFile["events"], chunkFile["events"])
self.assertEqual(foundFile["size"], chunkFile["size"])
self.assertEqual(len(foundFile["runs"]),
len(chunkFile["runs"]),
"Error: Wrong number of runs.")
for run in foundFile["runs"]:
runMatch = False
for chunkRun in chunkFile["runs"]:
if chunkRun.run == run.run and chunkRun.lumis == run.lumis:
runMatch = True
break
self.assertTrue(runMatch,
"Error: Run information is wrong.")
del goldenFiles[chunk["files"]]
singleChunk = dcs.singleChunkFileset("ACDCTest", "/ACDCTest/reco")
self.assertEqual(
singleChunk, {
"offset": 0,
"files": 11,
"events": 11264,
"lumis": 22,
"locations":
{"castor.cern.ch", "cmssrm.fnal.gov", "srm.ral.uk"}
}, "Error: Single chunk metadata is wrong")
return
