def singleChunk(self, acdc, acdcInfo, task):
"""Return a single block (inside a list) with all associated ACDC records"""
result = []
acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'],
acdcInfo['fileset'],
user = self.wmspec.getOwner().get("name"),
group = self.wmspec.getOwner().get("group"))
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
acdcBlock['offset'], acdcBlock['files'])
dbsBlock['NumberOfFiles'] = acdcBlock['files']
dbsBlock['NumberOfEvents'] = acdcBlock['events']
dbsBlock['NumberOfLumis'] = acdcBlock['lumis']
if task.inputLocationFlag():
dbsBlock["Sites"] = self.sites
else:
#TODO remove this line when all DBS origin_site_name is converted to PNN
acdcBlock["locations"] = self.siteDB.checkAndConvertSENameToPNN(acdcBlock["locations"])
#upto this
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(acdcBlock["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
result.append(dbsBlock)
return result
def fixedSizeChunk(self, acdc, acdcInfo, task):
"""Return a set of blocks with a fixed number of ACDC records"""
fixedSizeBlocks = []
chunkSize = 250
acdcBlocks = acdc.chunkFileset(acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize,
user = self.wmspec.getOwner().get("name"),
group = self.wmspec.getOwner().get("group"))
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'], block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
if task.inputLocationFlag():
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
fixedSizeBlocks.append(dbsBlock)
return fixedSizeBlocks
def singleChunk(self, acdc, acdcInfo, task):
"""Return a single block (inside a list) with all associated ACDC records"""
result = []
acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'],
acdcInfo['fileset'])
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
acdcBlock['offset'],
acdcBlock['files'])
dbsBlock['NumberOfFiles'] = acdcBlock['files']
dbsBlock['NumberOfEvents'] = acdcBlock['events']
dbsBlock['NumberOfLumis'] = acdcBlock['lumis']
if task.getTrustSitelists().get('trustlists'):
dbsBlock["Sites"] = self.sites
else:
if self.cric:
dbsBlock["Sites"] = self.cric.PNNstoPSNs(
acdcBlock["locations"])
else:
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(
acdcBlock["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
result.append(dbsBlock)
return result
def fixedSizeChunk(self, acdc, acdcInfo, task):
"""Return a set of blocks with a fixed number of ACDC records"""
fixedSizeBlocks = []
chunkSize = 250
acdcBlocks = acdc.chunkFileset(acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize,
user = self.wmspec.getOwner().get("name"),
group = self.wmspec.getOwner().get("group"))
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'], block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
if task.inputLocationFlag():
dbsBlock["Sites"] = self.sites
else:
#TODO remove this line when all DBS origin_site_name is converted to PNN
block["locations"] = self.siteDB.checkAndConvertSENameToPNN(block["locations"])
#upto this
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
fixedSizeBlocks.append(dbsBlock)
return fixedSizeBlocks
def fixedSizeChunk(self, acdc, acdcInfo, task):
"""Return a set of blocks with a fixed number of ACDC records"""
fixedSizeBlocks = []
chunkSize = 250
acdcBlocks = acdc.chunkFileset(acdcInfo['collection'],
acdcInfo['fileset'], chunkSize)
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'], block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
if task.getTrustSitelists().get('trustlists'):
dbsBlock["Sites"] = self.sites
else:
if self.cric:
dbsBlock["Sites"] = self.cric.PNNstoPSNs(
block["locations"])
else:
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(
block["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
fixedSizeBlocks.append(dbsBlock)
return fixedSizeBlocks
def singleChunk(self, acdc, acdcInfo, task):
"""Return a single block (inside a list) with all associated ACDC records"""
result = []
acdcBlock = acdc.singleChunkFileset(
acdcInfo['collection'],
acdcInfo['fileset'],
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
acdcBlock['offset'],
acdcBlock['files'])
dbsBlock['NumberOfFiles'] = acdcBlock['files']
dbsBlock['NumberOfEvents'] = acdcBlock['events']
dbsBlock['NumberOfLumis'] = acdcBlock['lumis']
if task.inputLocationFlag():
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = sitesFromStorageEelements(
acdcBlock["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
result.append(dbsBlock)
return result
def fixedSizeChunk(self, acdc, acdcInfo, task):
"""Return a set of blocks with a fixed number of ACDC records"""
fixedSizeBlocks = []
chunkSize = 250
acdcBlocks = acdc.chunkFileset(
acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize,
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'], block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
if task.inputLocationFlag():
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = sitesFromStorageEelements(
block["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
fixedSizeBlocks.append(dbsBlock)
return fixedSizeBlocks
def validBlocks(self, task):
"""Return blocks that pass the input data restriction"""
validBlocks = []
# TODO take the chunk size from parameter
chunkSize = 200
acdcInfo = task.getInputACDC()
if not acdcInfo:
raise WorkQueueWMSpecError(
self.wmspec, 'No acdc section for %s' % task.getPathName())
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
if self.data:
acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0])
else:
#if self.data is not passed, assume the the data is input dataset
# from the spec
acdcBlockSplit = False
if acdcBlockSplit:
dbsBlock = {}
dbsBlock['Name'] = self.data.keys()[0]
block = acdc.getChunkInfo(
acdcInfo['collection'],
acdcBlockSplit['TaskName'],
acdcBlockSplit['Offset'],
acdcBlockSplit['NumOfFiles'],
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock['ACDC'] = acdcInfo
dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"])
validBlocks.append(dbsBlock)
else:
acdcBlocks = acdc.chunkFileset(
acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize,
user=self.wmspec.getOwner().get("name"),
group=self.wmspec.getOwner().get("group"))
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'],
block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock["Sites"] = sitesFromStorageEelements(
block["locations"])
dbsBlock['ACDC'] = acdcInfo
validBlocks.append(dbsBlock)
return validBlocks
def validBlocks(self, task):
"""Return blocks that pass the input data restriction"""
validBlocks = []
# TODO take the chunk size from parameter
chunkSize = 200
acdcInfo = task.getInputACDC()
if not acdcInfo:
raise WorkQueueWMSpecError(self.wmspec, 'No acdc section for %s' % task.getPathName())
acdc = DataCollectionService(acdcInfo["server"], acdcInfo["database"])
if self.data:
acdcBlockSplit = ACDCBlock.splitBlockName(self.data.keys()[0])
else:
#if self.data is not passed, assume the the data is input dataset
# from the spec
acdcBlockSplit = False
if acdcBlockSplit:
dbsBlock = {}
dbsBlock['Name'] = self.data.keys()[0]
block = acdc.getChunkInfo(acdcInfo['collection'],
acdcBlockSplit['TaskName'],
acdcBlockSplit['Offset'],
acdcBlockSplit['NumOfFiles'],
user = self.wmspec.getOwner().get("name"),
group = self.wmspec.getOwner().get("group"))
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock['ACDC'] = acdcInfo
dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"])
validBlocks.append(dbsBlock)
else:
acdcBlocks = acdc.chunkFileset(acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize,
user = self.wmspec.getOwner().get("name"),
group = self.wmspec.getOwner().get("group"))
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'], block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
dbsBlock["Sites"] = sitesFromStorageEelements(block["locations"])
dbsBlock['ACDC'] = acdcInfo
validBlocks.append(dbsBlock)
return validBlocks
def singleChunk(self, acdc, acdcInfo, task):
"""Return a single block (inside a list) with all associated ACDC records"""
result = []
acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'],
acdcInfo['fileset'])
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
acdcBlock['offset'], acdcBlock['files'])
dbsBlock['NumberOfFiles'] = acdcBlock['files']
dbsBlock['NumberOfEvents'] = acdcBlock['events']
dbsBlock['NumberOfLumis'] = acdcBlock['lumis']
if task.getTrustSitelists().get('trustlists'):
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(acdcBlock["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
result.append(dbsBlock)
return result
def singleChunk(self, acdc, acdcInfo, task):
"""Return a single block (inside a list) with all associated ACDC records"""
result = []
acdcBlock = acdc.singleChunkFileset(acdcInfo['collection'],
acdcInfo['fileset'],
user = self.wmspec.getOwner().get("name"),
group = self.wmspec.getOwner().get("group"))
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
acdcBlock['offset'], acdcBlock['files'])
dbsBlock['NumberOfFiles'] = acdcBlock['files']
dbsBlock['NumberOfEvents'] = acdcBlock['events']
dbsBlock['NumberOfLumis'] = acdcBlock['lumis']
if task.inputLocationFlag():
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = sitesFromStorageEelements(acdcBlock["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
result.append(dbsBlock)
return result
def fixedSizeChunk(self, acdc, acdcInfo, task):
"""Return a set of blocks with a fixed number of ACDC records"""
fixedSizeBlocks = []
chunkSize = 250
acdcBlocks = acdc.chunkFileset(acdcInfo['collection'],
acdcInfo['fileset'],
chunkSize)
for block in acdcBlocks:
dbsBlock = {}
dbsBlock['Name'] = ACDCBlock.name(self.wmspec.name(),
acdcInfo["fileset"],
block['offset'], block['files'])
dbsBlock['NumberOfFiles'] = block['files']
dbsBlock['NumberOfEvents'] = block['events']
dbsBlock['NumberOfLumis'] = block['lumis']
if task.getTrustSitelists().get('trustlists'):
dbsBlock["Sites"] = self.sites
else:
dbsBlock["Sites"] = self.siteDB.PNNstoPSNs(block["locations"])
dbsBlock['ACDC'] = acdcInfo
if dbsBlock['NumberOfFiles']:
fixedSizeBlocks.append(dbsBlock)
return fixedSizeBlocks
