def _blockGenerator(self, dataset):
if dataset.startswith('/' + Globals.NOT_EXIST_DATASET):
raise NoDatasetError, "no dataset"
blocks = []
numOfEvents = GlobalParams.numOfFilesPerBlock(
) * GlobalParams.numOfEventsPerFile()
for i in range(GlobalParams.numOfBlocksPerDataset()):
blockName = "%s#%s" % (dataset, i + 1)
size = GlobalParams.numOfFilesPerBlock() * GlobalParams.sizeOfFile(
)
blocks.append({
'block_name':
blockName,
'NumberOfEvents':
numOfEvents,
'NumberOfFiles':
GlobalParams.numOfFilesPerBlock(),
'NumberOfLumis':
GlobalParams.numOfLumisPerBlock(),
'file_count':
GlobalParams.numOfFilesPerBlock(),
'Size':
size,
'open_for_writing':
'1' if self._openForWriting() else '0'
})
return blocks
def _fileGenerator(self, blockName, parentFlag):
files = []
for fileID in range(GlobalParams.numOfFilesPerBlock()):
fileName = "/store/data%s/file%s" % (blockName, fileID)
#Not sure why fileName is unit code - change to str
fileName = str(fileName)
parentFileName = "/store/data%s_parent/file%s_parent" % (blockName,
fileID)
#Not sure why fileName is unit code - change to str
parentFileName = str(parentFileName)
if parentFlag:
parentList = [
self._createDBSFile(blockName,
{'LogicalFileName': parentFileName})
]
else:
parentList = []
dbsFile = {
'LogicalFileName': fileName,
'ParentList': parentList,
}
files.append(self._createDBSFile(blockName, dbsFile))
return files
def _createDBSFile(self, blockName, dbsFile={}):
run = GlobalParams.getRunNumberForBlock(blockName)
defaultDBSFile = {
'Checksum':
"123456",
'NumberOfEvents':
GlobalParams.numOfEventsPerFile(),
'FileSize':
GlobalParams.sizeOfFile(),
'ParentList': [],
# assign run and lumi numbers in appropriate range: note run numbers
# for successive blocks may overlap if numOfRunsPerFile() > 1
'LumiList': [
{
'RunNumber':
int(
round((1. * lumi *
(GlobalParams.numOfRunsPerFile() - 1) /
GlobalParams.numOfLumisPerBlock()) + run)
), #random.randint(run, run + GlobalParams.numOfRunsPerFile() -1),
'LumiSectionNumber':
run * (GlobalParams.numOfLumisPerBlock()) + lumi - 1
} for lumi in range(GlobalParams.numOfLumisPerBlock())
]
}
defaultDBSFile.update(dbsFile)
return defaultDBSFile
def _blockGenerator(self, dataset):
if dataset.startswith('/' + Globals.NOT_EXIST_DATASET):
raise NoDatasetError, "no dataset"
blocks = []
numOfEvents = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfEventsPerFile()
for i in range(GlobalParams.numOfBlocksPerDataset()):
blockName = "%s#%s" % (dataset, i+1)
size = GlobalParams.numOfFilesPerBlock() * GlobalParams.sizeOfFile()
blocks.append(
{'block_name' : blockName,
'NumberOfEvents' : numOfEvents,
'NumberOfFiles' : GlobalParams.numOfFilesPerBlock(),
'NumberOfLumis' : GlobalParams.numOfLumisPerBlock(),
'file_count' : GlobalParams.numOfFilesPerBlock(),
'Size' : size,
'open_for_writing' : '1' if self._openForWriting() else '0'}
)
return blocks
def _createDBSFile(self, blockName, dbsFile = {}):
run = GlobalParams.getRunNumberForBlock(blockName)
defaultDBSFile = {'Checksum': "123456",
'NumberOfEvents': GlobalParams.numOfEventsPerFile(),
'FileSize': GlobalParams.sizeOfFile(),
'ParentList': [],
# assign run and lumi numbers in appropriate range: note run numbers
# for successive blocks may overlap if numOfRunsPerFile() > 1
'LumiList': [{'RunNumber': int(round((1. * lumi * (GlobalParams.numOfRunsPerFile() - 1) / GlobalParams.numOfLumisPerBlock()) + run)), #random.randint(run, run + GlobalParams.numOfRunsPerFile() -1),
'LumiSectionNumber': run*(GlobalParams.numOfLumisPerBlock()) + lumi -1}
for lumi in range(GlobalParams.numOfLumisPerBlock())]
}
defaultDBSFile.update(dbsFile)
return defaultDBSFile
def _blockGenerator(self, dataset):
if dataset.startswith('/' + Globals.NOT_EXIST_DATASET):
raise NoDatasetError, "no dataset"
blocks = []
numOfEvents = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfEventsPerFile()
for i in range(GlobalParams.numOfBlocksPerDataset()):
blockName = "%s#%s" % (dataset, i+1)
size = GlobalParams.numOfFilesPerBlock() * GlobalParams.sizeOfFile()
blocks.append(
{'Name' : blockName,
'NumberOfEvents' : numOfEvents,
'NumberOfFiles' : GlobalParams.numOfFilesPerBlock(),
'NumberOfLumis' : GlobalParams.numOfLumisPerBlock(),
'Size' : size,
'Parents' : ()}
)
return blocks
def _fileGenerator(self, blockName, parentFlag):
files = []
for fileID in range(GlobalParams.numOfFilesPerBlock()):
fileName = "/store/data%s/file%s" % (blockName, fileID)
#Not sure why fileName is unit code - change to str
fileName = str(fileName)
parentFileName = "/store/data%s_parent/file%s_parent" % (blockName, fileID)
#Not sure why fileName is unit code - change to str
parentFileName = str(parentFileName)
if parentFlag:
parentList = [self._createDBSFile(blockName, {'LogicalFileName':parentFileName})]
else:
parentList = []
dbsFile = {'LogicalFileName': fileName,
'ParentList' : parentList,
}
files.append(self._createDBSFile(blockName, dbsFile))
return files
def getParentBlock(self, block, numberOfParents = 1):
blocks = []
numOfEvents = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfEventsPerFile()
for i in range(numberOfParents):
blockName = "%s_parent_%s" % (block, i+1)
size = GlobalParams.numOfFilesPerBlock() * GlobalParams.sizeOfFile()
blocks.append({'Name' : blockName,
'NumberOfEvents' : numOfEvents,
'NumberOfFiles' : GlobalParams.numOfFilesPerBlock(),
'NumberOfLumis' : GlobalParams.numOfLumisPerBlock(),
'Size' : size,
'StorageElementList':[{'Role' : '', 'Name' : x} for x in \
self.getLocation(blockName)],
'Parents' : ()}
)
return blocks
def getParentBlock(self, block, numberOfParents = 1):
blocks = []
numOfEvents = GlobalParams.numOfFilesPerBlock() * GlobalParams.numOfEventsPerFile()
for i in range(numberOfParents):
dataset, blockname = block.split('#') # append parent block id to tier
blockName = "%s_parent_%s#%s" % (dataset, i+1, blockname)
size = GlobalParams.numOfFilesPerBlock() * GlobalParams.sizeOfFile()
blocks.append({'Name' : blockName,
'NumberOfEvents' : numOfEvents,
'NumberOfFiles' : GlobalParams.numOfFilesPerBlock(),
'NumberOfLumis' : GlobalParams.numOfLumisPerBlock(),
'Size' : size,
'StorageElementList' : self.getLocation(blockName),
'Parents' : ()}
)
return blocks
def _createDBSFile(self, blockName, dbsFile = {}):
run = GlobalParams.getRunNumberForBlock(blockName)
defaultDBSFile = {'Checksum': "123456",
'NumberOfEvents': GlobalParams.numOfEventsPerFile(),
'FileSize': GlobalParams.sizeOfFile(),
'ParentList': [],
}
lumiList = []
for run in xrange(1, GlobalParams.numOfRunsPerFile() + 1):
lumiList.append(
{run:[run*(GlobalParams.numOfLumisPerBlock()) + lumi -1
for lumi in range(GlobalParams.numOfLumisPerBlock())]
})
defaultDBSFile.update({'LumiList':lumiList})
defaultDBSFile.update(dbsFile)
return defaultDBSFile
def getParentBlock(self, block, numberOfParents=1):
blocks = []
numOfEvents = GlobalParams.numOfFilesPerBlock(
) * GlobalParams.numOfEventsPerFile()
for i in range(numberOfParents):
dataset, blockname = block.split(
'#') # append parent block id to tier
blockName = "%s_parent_%s#%s" % (dataset, i + 1, blockname)
size = GlobalParams.numOfFilesPerBlock() * GlobalParams.sizeOfFile(
)
blocks.append({
'Name': blockName,
'NumberOfEvents': numOfEvents,
'NumberOfFiles': GlobalParams.numOfFilesPerBlock(),
'NumberOfLumis': GlobalParams.numOfLumisPerBlock(),
'Size': size,
'StorageElementList': self.getLocation(blockName),
'Parents': ()
})
return blocks
def _createDBSFile(self, blockName, dbsFile={}):
run = GlobalParams.getRunNumberForBlock(blockName)
defaultDBSFile = {
'Checksum': "123456",
'NumberOfEvents': GlobalParams.numOfEventsPerFile(),
'FileSize': GlobalParams.sizeOfFile(),
'ParentList': [],
}
lumiList = []
for run in xrange(1, GlobalParams.numOfRunsPerFile() + 1):
lumiList.append({
run: [
run * (GlobalParams.numOfLumisPerBlock()) + lumi - 1
for lumi in range(GlobalParams.numOfLumisPerBlock())
]
})
defaultDBSFile.update({'LumiList': lumiList})
defaultDBSFile.update(dbsFile)
return defaultDBSFile
def _openForWriting(self):
"""Is block open or closed?
Should do this on a block by block basis but so far not needed,
just make a global state"""
return GlobalParams.blocksOpenForWriting()
def _openForWriting(self): """Is block open or closed? Should do this on a block by block basis but so far not needed, just make a global state""" return GlobalParams.blocksOpenForWriting()
