def createWMBSHelperWithTopTask(self, wmspec, block, mask = None):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec, block, mask, cachepath = self.workDir)
if block:
block = self.dbs.getFileBlock(block)[block]
wmbs.createSubscriptionAndAddFiles(block = block)
return wmbs
def addNewFilesToOpenSubscriptions(self, *elements):
"""Inject new files to wmbs for running elements that have new files.
Assumes elements are from the same workflow"""
if not self.params['LocalQueueFlag']:
return
wmspec = None
for ele in elements:
if not ele.isRunning() or not ele['SubscriptionId'] or not ele:
continue
if not ele['Inputs'] or not ele['OpenForNewData']:
continue
if not wmspec:
wmspec = self.backend.getWMSpec(ele['RequestName'])
blockName, dbsBlock = self._getDBSBlock(ele, wmspec)
if ele['NumOfFilesAdded'] != len(dbsBlock['Files']):
self.logger.info("Adding new files to open block %s (%s)" %
(blockName, ele.id))
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
wmbsHelper = WMBSHelper(wmspec, ele['TaskName'], blockName,
ele['Mask'], self.params['CacheDir'])
ele['NumOfFilesAdded'] += wmbsHelper.createSubscriptionAndAddFiles(
block=dbsBlock)[1]
self.backend.updateElements(
ele.id, NumOfFilesAdded=ele['NumOfFilesAdded'])
if dbsBlock['IsOpen'] != ele['OpenForNewData']:
self.logger.info("Closing open block %s (%s)" %
(blockName, ele.id))
self.backend.updateInboxElements(
ele['ParentQueueId'], OpenForNewData=dbsBlock['IsOpen'])
self.backend.updateElements(ele.id,
OpenForNewData=dbsBlock['IsOpen'])
ele['OpenForNewData'] = dbsBlock['IsOpen']
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
"""Inject data into wmbs and create subscription.
"""
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
self.logger.info("Adding WMBS subscription for %s" %
match['RequestName'])
mask = match['Mask']
wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask,
self.params['CacheDir'])
sub, match[
'NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(
block=dbsBlock)
self.logger.info(
"Created top level subscription %s for %s with %s files" %
(sub['id'], match['RequestName'], match['NumOfFilesAdded']))
# update couch with wmbs subscription info
match['SubscriptionId'] = sub['id']
match['Status'] = 'Running'
# do update rather than save to avoid conflicts from other thread writes
self.backend.updateElements(match.id,
Status='Running',
SubscriptionId=sub['id'],
NumOfFilesAdded=match['NumOfFilesAdded'])
return sub
def createWMBSHelperWithTopTask(self,
wmspec,
block,
mask=None,
parentFlag=False,
detail=False,
commonLocation=None):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec,
topLevelTask.name(),
block,
mask,
cachepath=self.workDir,
commonLocation=commonLocation)
if block:
blockName = block
if parentFlag:
block = self.dbs.getFileBlockWithParents(blockName)
location = self.phedex.getReplicaPhEDExNodesForBlocks(
block=[blockName], complete='y')
block['PhEDExNodeNames'] = location[blockName]
else:
block = self.dbs.getFileBlock(blockName)
location = self.phedex.getReplicaPhEDExNodesForBlocks(
block=[blockName], complete='y')
block['PhEDExNodeNames'] = location[blockName]
sub, files = wmbs.createSubscriptionAndAddFiles(block=block)
if detail:
return wmbs, sub, files
else:
return wmbs
def addNewFilesToOpenSubscriptions(self, *elements):
"""Inject new files to wmbs for running elements that have new files.
Assumes elements are from the same workflow"""
if not self.params['LocalQueueFlag']:
return
wmspec = None
for ele in elements:
if not ele.isRunning() or not ele['SubscriptionId'] or not ele:
continue
if not ele['Inputs'] or not ele['OpenForNewData']:
continue
if not wmspec:
wmspec = self.backend.getWMSpec(ele['RequestName'])
blockName, dbsBlock = self._getDBSBlock(ele, wmspec)
if ele['NumOfFilesAdded'] != len(dbsBlock['Files']):
self.logger.info("Adding new files to open block %s (%s)" % (blockName, ele.id))
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
wmbsHelper = WMBSHelper(wmspec, ele['TaskName'], blockName, ele['Mask'], self.params['CacheDir'])
ele['NumOfFilesAdded'] += wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)[1]
self.backend.updateElements(ele.id, NumOfFilesAdded = ele['NumOfFilesAdded'])
if dbsBlock['IsOpen'] != ele['OpenForNewData']:
self.logger.info("Closing open block %s (%s)" % (blockName, ele.id))
self.backend.updateInboxElements(ele['ParentQueueId'], OpenForNewData = dbsBlock['IsOpen'])
self.backend.updateElements(ele.id, OpenForNewData = dbsBlock['IsOpen'])
ele['OpenForNewData'] = dbsBlock['IsOpen']
def main():
start = time.time()
# blockName = match['Inputs'].keys()[0]
blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055"
# acdcInfo = match['ACDC']
acdcInfo = {"database": "acdcserver",
"fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0",
"collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222",
"server": "https://cmsweb.cern.ch/couchdb"}
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
splitedBlockName = ACDCBlock.splitBlockName(blockName)
print("Splitted block name: %s" % splitedBlockName)
fileLists = acdc.getChunkFiles(acdcInfo['collection'],
acdcInfo['fileset'],
splitedBlockName['Offset'],
splitedBlockName['NumOfFiles'])
print("Retrieved %d unique files from the ACDCServer" % len(fileLists))
block = {}
block["Files"] = fileLists
wantedLumis = set([252052, 240646])
for f in fileLists:
for run in f['runs']:
maskDict = run.json()
lumisSet = set(maskDict['Lumis'].keys())
if wantedLumis.intersection(lumisSet):
print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet)))
# with open("chunkfiles.json", 'w') as fo:
# json.dump(block, fo)
end = time.time()
print("Spent %s secs running so far" % (end - start))
sys.exit(1)
### Now doing the WMBSHelper stuff
reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache"
requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376"
wmspec = WMWorkloadHelper()
wmspec.loadSpecFromCouch(reqUrl, requestName)
taskName = "HIG-RunIIFall17DRPremix-00788_0"
mask = None
cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache"
# wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir'])
wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir)
sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def createWMBSHelperWithTopTask(self, wmspec, block, mask=None,
parentFlag=False, detail=False):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath=self.workDir)
if block:
if parentFlag:
block = self.dbs.getFileBlockWithParents(block)[block]
else:
block = self.dbs.getFileBlock(block)[block]
sub, files = wmbs.createSubscriptionAndAddFiles(block=block)
if detail:
return wmbs, sub, files
else:
return wmbs
def createWMBSHelperWithTopTask(self, wmspec, block, mask = None,
parentFlag = False, detail = False):
topLevelTask = getFirstTask(wmspec)
wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath = self.workDir)
if block:
if parentFlag:
block = self.dbs.getFileBlockWithParents(block)[block]
else:
block = self.dbs.getFileBlock(block)[block]
sub, files = wmbs.createSubscriptionAndAddFiles(block = block)
if detail:
return wmbs, sub, files
else:
return wmbs
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
"""Inject data into wmbs and create subscription.
"""
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
self.logger.info("Adding WMBS subscription for %s" % match['RequestName'])
mask = match['Mask']
wmbsHelper = WMBSHelper(wmspec, blockName, mask, self.params['CacheDir'])
sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)
self.logger.info("Created top level subscription %s for %s with %s files" % (sub['id'],
match['RequestName'],
match['NumOfFilesAdded']))
match['SubscriptionId'] = sub['id']
match['Status'] = 'Running'
self.backend.saveElements(match)
return sub
def _wmbsPreparation(self, match, wmspec, blockName, dbsBlock):
"""Inject data into wmbs and create subscription.
"""
from WMCore.WorkQueue.WMBSHelper import WMBSHelper
self.logger.info("Adding WMBS subscription for %s" % match['RequestName'])
mask = match['Mask']
wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir'])
sub, match['NumOfFilesAdded'] = wmbsHelper.createSubscriptionAndAddFiles(block = dbsBlock)
self.logger.info("Created top level subscription %s for %s with %s files" % (sub['id'],
match['RequestName'],
match['NumOfFilesAdded']))
# update couch with wmbs subscription info
match['SubscriptionId'] = sub['id']
match['Status'] = 'Running'
# do update rather than save to avoid conflicts from other thread writes
self.backend.updateElements(match.id, Status = 'Running', SubscriptionId = sub['id'],
NumOfFilesAdded = match['NumOfFilesAdded'])
return sub
