def validBlocks(self, task): """Return blocks that pass the input data restriction""" validBlocks = [] # TODO take the chunk size from parameter chunkSize = 200 acdcInfo = task.getInputACDC() if not acdcInfo: raise WorkQueueWMSpecError( self.wmspec, 'No acdc section for %s' % task.getPathName()) acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) if self.data: acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0]) else: #if self.data is not passed, assume the the data is input dataset # from the spec acdcBlockSplit = False if acdcBlockSplit: dbsBlock = {} dbsBlock['Name'] = self.data.keys()[0] block = acdc.getChunkInfo( acdcInfo['collection'], acdcBlockSplit['TaskName'], acdcBlockSplit['Offset'], acdcBlockSplit['NumOfFiles'], user=self.wmspec.getOwner().get("name"), group=self.wmspec.getOwner().get("group")) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock['ACDC'] = acdcInfo dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"]) validBlocks.append(dbsBlock) else: acdcBlocks = acdc.chunkFileset( acdcInfo['collection'], acdcInfo['fileset'], chunkSize, user=self.wmspec.getOwner().get("name"), group=self.wmspec.getOwner().get("group")) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock["Sites"] = sitesFromStorageEelements( block["locations"]) dbsBlock['ACDC'] = acdcInfo validBlocks.append(dbsBlock) return validBlocks
def validBlocks(self, task): """Return blocks that pass the input data restriction""" validBlocks = [] # TODO take the chunk size from parameter chunkSize = 200 acdcInfo = task.getInputACDC() if not acdcInfo: raise WorkQueueWMSpecError(self.wmspec, 'No acdc section for %s' % task.getPathName()) acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) if self.data: acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0]) else: #if self.data is not passed, assume the the data is input dataset # from the spec acdcBlockSplit = False if acdcBlockSplit: dbsBlock = {} dbsBlock['Name'] = self.data.keys()[0] block = acdc.getChunkInfo(acdcInfo['collection'], acdcBlockSplit['TaskName'], acdcBlockSplit['Offset'], acdcBlockSplit['NumOfFiles'], user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock['ACDC'] = acdcInfo dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"]) validBlocks.append(dbsBlock) else: acdcBlocks = acdc.chunkFileset(acdcInfo['collection'], acdcInfo['fileset'], chunkSize, user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"]) dbsBlock['ACDC'] = acdcInfo validBlocks.append(dbsBlock) return validBlocks
def singleChunk(self, acdc, acdcInfo, task): """Return a single block (inside a list) with all associated ACDC records""" result = [] acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'], acdcInfo['fileset']) dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], acdcBlock['offset'], acdcBlock['files']) dbsBlock['NumberOfFiles'] = acdcBlock['files'] dbsBlock['NumberOfEvents'] = acdcBlock['events'] dbsBlock['NumberOfLumis'] = acdcBlock['lumis'] if task.getTrustSitelists().get('trustlists'): dbsBlock["Sites"] = self.sites else: if self.cric: dbsBlock["Sites"] = self.cric.PNNstoPSNs( acdcBlock["locations"]) else: dbsBlock["Sites"] = self.siteDB.PNNstoPSNs( acdcBlock["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: result.append(dbsBlock) return result
def fixedSizeChunk(self, acdc, acdcInfo, task): """Return a set of blocks with a fixed number of ACDC records""" fixedSizeBlocks = [] chunkSize = 250 acdcBlocks = acdc.chunkFileset(acdcInfo['collection'], acdcInfo['fileset'], chunkSize) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] if task.getTrustSitelists().get('trustlists'): dbsBlock["Sites"] = self.sites else: if self.cric: dbsBlock["Sites"] = self.cric.PNNstoPSNs( block["locations"]) else: dbsBlock["Sites"] = self.siteDB.PNNstoPSNs( block["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: fixedSizeBlocks.append(dbsBlock) return fixedSizeBlocks
def singleChunk(self, acdc, acdcInfo, task): """Return a single block (inside a list) with all associated ACDC records""" result = [] acdcBlock = acdc.singleChunkFileset( acdcInfo['collection'], acdcInfo['fileset'], user=self.wmspec.getOwner().get("name"), group=self.wmspec.getOwner().get("group")) dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], acdcBlock['offset'], acdcBlock['files']) dbsBlock['NumberOfFiles'] = acdcBlock['files'] dbsBlock['NumberOfEvents'] = acdcBlock['events'] dbsBlock['NumberOfLumis'] = acdcBlock['lumis'] if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = sitesFromStorageEelements( acdcBlock["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: result.append(dbsBlock) return result
def fixedSizeChunk(self, acdc, acdcInfo, task): """Return a set of blocks with a fixed number of ACDC records""" fixedSizeBlocks = [] chunkSize = 250 acdcBlocks = acdc.chunkFileset(acdcInfo['collection'], acdcInfo['fileset'], chunkSize, user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: #TODO remove this line when all DBS origin_site_name is converted to PNN block["locations"] = self.siteDB.checkAndConvertSENameToPNN(block["locations"]) #upto this dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: fixedSizeBlocks.append(dbsBlock) return fixedSizeBlocks
def _getDBSBlock(self, match, wmspec): """Get DBS info for this block""" blockName = match['Inputs'].keys()[0] #TODO: Allow more than one if match['ACDC']: acdcInfo = match['ACDC'] acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) collection = acdc.getDataCollection(acdcInfo['collection']) splitedBlockName = ACDCBlock.splitBlockName(blockName) fileLists = acdc.getChunkFiles(acdcInfo['collection'], acdcInfo['fileset'], splitedBlockName['Offset'], splitedBlockName['NumOfFiles'], user = wmspec.getOwner().get("name"), group = wmspec.getOwner().get("group")) block = {} block["Files"] = fileLists return blockName, block else: dbs = get_dbs(match['Dbs']) if wmspec.getTask(match['TaskName']).parentProcessingFlag(): dbsBlockDict = dbs.getFileBlockWithParents(blockName) else: dbsBlockDict = dbs.getFileBlock(blockName) if wmspec.locationDataSourceFlag(): blockInfo = dbsBlockDict[blockName] seElements = [] for cmsSite in match['Inputs'].values()[0]: #TODO: Allow more than one ses = self.SiteDB.cmsNametoSE(cmsSite) seElements.extend(ses) seElements = list(set(seElements)) blockInfo['StorageElements'] = seElements return blockName, dbsBlockDict[blockName]
def _getDBSBlock(self, match, wmspec): """Get DBS info for this block""" blockName = match['Inputs'].keys()[0] #TODO: Allow more than one if match['ACDC']: acdcInfo = match['ACDC'] acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) collection = acdc.getDataCollection(acdcInfo['collection']) splitedBlockName = ACDCBlock.splitBlockName(blockName) fileLists = acdc.getChunkFiles(acdcInfo['collection'], acdcInfo['fileset'], splitedBlockName['Offset'], splitedBlockName['NumOfFiles'], user = wmspec.getOwner().get("name"), group = wmspec.getOwner().get("group")) block = {} block["Files"] = fileLists return blockName, block else: dbs = get_dbs(match['Dbs']) if wmspec.getTask(match['TaskName']).parentProcessingFlag(): dbsBlockDict = dbs.getFileBlockWithParents(blockName) else: dbsBlockDict = dbs.getFileBlock(blockName) return blockName, dbsBlockDict[blockName]
def fixedSizeChunk(self, acdc, acdcInfo, task): """Return a set of blocks with a fixed number of ACDC records""" fixedSizeBlocks = [] chunkSize = 250 acdcBlocks = acdc.chunkFileset(acdcInfo['collection'], acdcInfo['fileset'], chunkSize, user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: fixedSizeBlocks.append(dbsBlock) return fixedSizeBlocks
def singleChunk(self, acdc, acdcInfo, task): """Return a single block (inside a list) with all associated ACDC records""" result = [] acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'], acdcInfo['fileset'], user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], acdcBlock['offset'], acdcBlock['files']) dbsBlock['NumberOfFiles'] = acdcBlock['files'] dbsBlock['NumberOfEvents'] = acdcBlock['events'] dbsBlock['NumberOfLumis'] = acdcBlock['lumis'] if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: #TODO remove this line when all DBS origin_site_name is converted to PNN acdcBlock["locations"] = self.siteDB.checkAndConvertSENameToPNN(acdcBlock["locations"]) #upto this dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(acdcBlock["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: result.append(dbsBlock) return result
def fixedSizeChunk(self, acdc, acdcInfo, task): """Return a set of blocks with a fixed number of ACDC records""" fixedSizeBlocks = [] chunkSize = 250 acdcBlocks = acdc.chunkFileset( acdcInfo['collection'], acdcInfo['fileset'], chunkSize, user=self.wmspec.getOwner().get("name"), group=self.wmspec.getOwner().get("group")) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = sitesFromStorageEelements( block["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: fixedSizeBlocks.append(dbsBlock) return fixedSizeBlocks
def organiseByDbs(self, dataItems): """Sort items by dbs instances - return dict with DBSReader as key & data items as values""" itemsByDbs = defaultdict(list) for item in dataItems: if ACDCBlock.checkBlockName(item['name']): # if it is acdc block don't update location. location should be # inserted when block is queued and not supposed to change continue itemsByDbs[get_dbs(item['dbs_url'])].append(item['name']) return itemsByDbs
def main(): start = time.time() # blockName = match['Inputs'].keys()[0] blockName = "/acdc/vlimant_ACDC0_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_190218_145226_481/:pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222:SUS-RunIIFall18wmLHEGS-00025_0/0/31055" # acdcInfo = match['ACDC'] acdcInfo = {"database": "acdcserver", "fileset": "/pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222/SUS-RunIIFall18wmLHEGS-00025_0", "collection": "pdmvserv_task_SUS-RunIIFall18wmLHEGS-00025__v1_T_181211_005112_2222", "server": "https://cmsweb.cern.ch/couchdb"} acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) splitedBlockName = ACDCBlock.splitBlockName(blockName) print("Splitted block name: %s" % splitedBlockName) fileLists = acdc.getChunkFiles(acdcInfo['collection'], acdcInfo['fileset'], splitedBlockName['Offset'], splitedBlockName['NumOfFiles']) print("Retrieved %d unique files from the ACDCServer" % len(fileLists)) block = {} block["Files"] = fileLists wantedLumis = set([252052, 240646]) for f in fileLists: for run in f['runs']: maskDict = run.json() lumisSet = set(maskDict['Lumis'].keys()) if wantedLumis.intersection(lumisSet): print("File: %s with events: %s, contains these lumis: %s" % (f['lfn'], f['events'], wantedLumis.intersection(lumisSet))) # with open("chunkfiles.json", 'w') as fo: # json.dump(block, fo) end = time.time() print("Spent %s secs running so far" % (end - start)) sys.exit(1) ### Now doing the WMBSHelper stuff reqUrl = "https://cmsweb.cern.ch/couchdb/reqmgr_workload_cache" requestName = "vlimant_ACDC0_task_HIG-RunIIFall17wmLHEGS-01122__v1_T_180808_130708_5376" wmspec = WMWorkloadHelper() wmspec.loadSpecFromCouch(reqUrl, requestName) taskName = "HIG-RunIIFall17DRPremix-00788_0" mask = None cacheDir = "/data/srv/wmagent/v1.1.14.patch6/install/wmagent/WorkQueueManager/cache" # wmbsHelper = WMBSHelper(wmspec, match['TaskName'], blockName, mask, self.params['CacheDir']) wmbsHelper = WMBSHelper(wmspec, taskName, blockName, mask, cacheDir) sub, numFilesAdded = wmbsHelper.createSubscriptionAndAddFiles(block=block)
def validBlocks(self, task): """Return blocks that pass the input data restriction according to the splitting algorithm""" validBlocks = [] acdcInfo = task.getInputACDC() if not acdcInfo: raise WorkQueueWMSpecError( self.wmspec, 'No acdc section for %s' % task.getPathName()) acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) if self.data: acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0]) else: # if self.data is not passed, assume the the data is input dataset # from the spec acdcBlockSplit = False if acdcBlockSplit: dbsBlock = {} dbsBlock['Name'] = self.data.keys()[0] block = acdc.getChunkInfo( acdcInfo['collection'], acdcBlockSplit['TaskName'], acdcBlockSplit['Offset'], acdcBlockSplit['NumOfFiles'], user=self.wmspec.getOwner().get("name"), group=self.wmspec.getOwner().get("group")) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock['ACDC'] = acdcInfo if task.getTrustSitelists(): dbsBlock["Sites"] = self.sites else: # TODO remove this line when all DBS origin_site_name is converted to PNN block["locations"] = self.siteDB.checkAndConvertSENameToPNN( block["locations"]) # upto this dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"]) validBlocks.append(dbsBlock) else: if self.args['SplittingAlgo'] in self.unsupportedAlgos: raise WorkQueueWMSpecError( self.wmspec, 'ACDC is not supported for %s' % self.args['SplittingAlgo']) splittingFunc = self.defaultAlgo if self.args['SplittingAlgo'] in self.algoMapping: splittingFunc = self.algoMapping[self.args['SplittingAlgo']] validBlocks = splittingFunc(acdc, acdcInfo, task) return validBlocks
def validBlocks(self, task): """Return blocks that pass the input data restriction according to the splitting algorithm""" validBlocks = [] acdcInfo = task.getInputACDC() if not acdcInfo: raise WorkQueueWMSpecError(self.wmspec, 'No acdc section for %s' % task.getPathName()) acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) if self.data: acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0]) else: # if self.data is not passed, assume the the data is input dataset # from the spec acdcBlockSplit = False if acdcBlockSplit: dbsBlock = {} dbsBlock['Name'] = self.data.keys()[0] block = acdc.getChunkInfo(acdcInfo['collection'], acdcBlockSplit['TaskName'], acdcBlockSplit['Offset'], acdcBlockSplit['NumOfFiles'], user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock['ACDC'] = acdcInfo if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: #TODO remove this line when all DBS origin_site_name is converted to PNN block["locations"] = self.siteDB.checkAndConvertSENameToPNN(block["locations"]) #upto this dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"]) validBlocks.append(dbsBlock) else: if self.args['SplittingAlgo'] in self.unsupportedAlgos: raise WorkQueueWMSpecError(self.wmspec, 'ACDC is not supported for %s' % self.args['SplittingAlgo']) splittingFunc = self.defaultAlgo if self.args['SplittingAlgo'] in self.algoMapping: splittingFunc = self.algoMapping[self.args['SplittingAlgo']] validBlocks = splittingFunc(acdc, acdcInfo, task) return validBlocks
def validBlocks(self, task): """Return blocks that pass the input data restriction according to the splitting algorithm""" validBlocks = [] acdcInfo = task.getInputACDC() if not acdcInfo: raise WorkQueueWMSpecError( self.wmspec, 'No acdc section for %s' % task.getPathName()) acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) if self.data: acdcBlockSplit = ACDCBlock.splitBlockName(next(iter(self.data))) else: # if self.data is not passed, assume the the data is input dataset # from the spec acdcBlockSplit = False if acdcBlockSplit: dbsBlock = {} dbsBlock['Name'] = next(iter(self.data)) block = acdc.getChunkInfo(acdcInfo['collection'], acdcBlockSplit['TaskName'], acdcBlockSplit['Offset'], acdcBlockSplit['NumOfFiles']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock['ACDC'] = acdcInfo if task.getTrustSitelists().get('trustlists'): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = self.cric.PNNstoPSNs(block["locations"]) validBlocks.append(dbsBlock) else: if self.args['SplittingAlgo'] in self.unsupportedAlgos: raise WorkQueueWMSpecError( self.wmspec, 'ACDC is not supported for %s' % self.args['SplittingAlgo']) splittingFunc = self.defaultAlgo if self.args['SplittingAlgo'] in self.algoMapping: splittingFunc = self.algoMapping[self.args['SplittingAlgo']] validBlocks = splittingFunc(acdc, acdcInfo, task) return validBlocks
def singleChunk(self, acdc, acdcInfo, task): """Return a single block (inside a list) with all associated ACDC records""" result = [] acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'], acdcInfo['fileset']) dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], acdcBlock['offset'], acdcBlock['files']) dbsBlock['NumberOfFiles'] = acdcBlock['files'] dbsBlock['NumberOfEvents'] = acdcBlock['events'] dbsBlock['NumberOfLumis'] = acdcBlock['lumis'] if task.getTrustSitelists().get('trustlists'): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(acdcBlock["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: result.append(dbsBlock) return result
def validBlocks(self, task): """Return blocks that pass the input data restriction according to the splitting algorithm""" validBlocks = [] acdcInfo = task.getInputACDC() if not acdcInfo: raise WorkQueueWMSpecError(self.wmspec, 'No acdc section for %s' % task.getPathName()) acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) if self.data: acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0]) else: # if self.data is not passed, assume the the data is input dataset # from the spec acdcBlockSplit = False if acdcBlockSplit: dbsBlock = {} dbsBlock['Name'] = self.data.keys()[0] block = acdc.getChunkInfo(acdcInfo['collection'], acdcBlockSplit['TaskName'], acdcBlockSplit['Offset'], acdcBlockSplit['NumOfFiles']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] dbsBlock['ACDC'] = acdcInfo if task.getTrustSitelists().get('trustlists'): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"]) validBlocks.append(dbsBlock) else: if self.args['SplittingAlgo'] in self.unsupportedAlgos: raise WorkQueueWMSpecError(self.wmspec, 'ACDC is not supported for %s' % self.args['SplittingAlgo']) splittingFunc = self.defaultAlgo if self.args['SplittingAlgo'] in self.algoMapping: splittingFunc = self.algoMapping[self.args['SplittingAlgo']] validBlocks = splittingFunc(acdc, acdcInfo, task) return validBlocks
def singleChunk(self, acdc, acdcInfo, task): """Return a single block (inside a list) with all associated ACDC records""" result = [] acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'], acdcInfo['fileset'], user = self.wmspec.getOwner().get("name"), group = self.wmspec.getOwner().get("group")) dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], acdcBlock['offset'], acdcBlock['files']) dbsBlock['NumberOfFiles'] = acdcBlock['files'] dbsBlock['NumberOfEvents'] = acdcBlock['events'] dbsBlock['NumberOfLumis'] = acdcBlock['lumis'] if task.inputLocationFlag(): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = sitesFromStorageEelements(acdcBlock["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: result.append(dbsBlock) return result
def fixedSizeChunk(self, acdc, acdcInfo, task): """Return a set of blocks with a fixed number of ACDC records""" fixedSizeBlocks = [] chunkSize = 250 acdcBlocks = acdc.chunkFileset(acdcInfo['collection'], acdcInfo['fileset'], chunkSize) for block in acdcBlocks: dbsBlock = {} dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(), acdcInfo["fileset"], block['offset'], block['files']) dbsBlock['NumberOfFiles'] = block['files'] dbsBlock['NumberOfEvents'] = block['events'] dbsBlock['NumberOfLumis'] = block['lumis'] if task.getTrustSitelists().get('trustlists'): dbsBlock["Sites"] = self.sites else: dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"]) dbsBlock['ACDC'] = acdcInfo if dbsBlock['NumberOfFiles']: fixedSizeBlocks.append(dbsBlock) return fixedSizeBlocks
def _getDBSBlock(self, match, wmspec): """Get DBS info for this block""" blockName = match['Inputs'].keys()[0] #TODO: Allow more than one if match['ACDC']: acdcInfo = match['ACDC'] acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"]) collection = acdc.getDataCollection(acdcInfo['collection']) splitedBlockName = ACDCBlock.splitBlockName(blockName) fileLists = acdc.getChunkFiles( acdcInfo['collection'], acdcInfo['fileset'], splitedBlockName['Offset'], splitedBlockName['NumOfFiles'], user=wmspec.getOwner().get("name"), group=wmspec.getOwner().get("group")) block = {} block["Files"] = fileLists return blockName, block else: dbs = get_dbs(match['Dbs']) if wmspec.getTask(match['TaskName']).parentProcessingFlag(): dbsBlockDict = dbs.getFileBlockWithParents(blockName) else: dbsBlockDict = dbs.getFileBlock(blockName) if wmspec.locationDataSourceFlag(): blockInfo = dbsBlockDict[blockName] seElements = [] for cmsSite in match['Inputs'].values( )[0]: #TODO: Allow more than one ses = self.SiteDB.cmsNametoSE(cmsSite) seElements.extend(ses) seElements = list(set(seElements)) blockInfo['StorageElements'] = seElements return blockName, dbsBlockDict[blockName]