def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded.
I will do this by DAS for now to break
the monstrous calls down into smaller chunks.
"""
# Grab all the Dataset-Algo combindations
dasList = self.dbsUtil.findUploadableDAS()
if len(dasList) < 1:
# Then there's nothing to do
return []
readyBlocks = []
for dasInfo in dasList:
dasID = dasInfo['DAS_ID']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(das = dasID)
except WMException:
raise
except Exception, ex:
msg = "Unhandled exception while loading uploadable files for DAS.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DAS being loaded: %s\n" % dasID)
raise DBSUploadException(msg)
# Get the blocks
if not dasID in self.dasCache.keys():
# Then we have a new DAS
# Add it
self.dasCache[dasID] = {}
dasBlocks = self.dasCache.get(dasID)
# Sort the files and blocks by location
fileDict = sortListByKey(input = loadedFiles, key = 'locations')
# Now we have both files and blocks
# We need a sorting algorithm of sorts...
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if len(files) < 1:
# Nothing to do here
continue
dasBlocks = self.dasCache[dasID].get(location, [])
if len(dasBlocks) > 0:
# Load from cache
currentBlock = self.blockCache.get(dasBlocks[0])
else:
blockname = '%s#%s' % (files[0]['datasetPath'], makeUUID())
currentBlock = DBSBlock(name = blockname,
location = location, das = dasID)
# Add the era info
currentBlock.setAcquisitionEra(era = dasInfo['AcquisitionEra'])
currentBlock.setProcessingVer(era = dasInfo['ProcessingVer'])
self.addNewBlock(block = currentBlock)
dasBlocks.append(currentBlock.getName())
for newFile in files:
if not newFile.get('block', 1) == None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile = newFile,
block = currentBlock):
# Then we have to close the block and get a new one
currentBlock.status = 'Pending'
readyBlocks.append(currentBlock)
dasBlocks.remove(currentBlock.getName())
currentBlock = self.getBlock(newFile = newFile,
dasBlocks = dasBlocks,
location = location,
das = dasID)
currentBlock.setAcquisitionEra(era = dasInfo['AcquisitionEra'])
currentBlock.setProcessingVer(era = dasInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(dbsFile = newFile)
self.filesToUpdate.append({'filelfn': newFile['lfn'],
'block': currentBlock.getName()})
# Done with the location
readyBlocks.append(currentBlock)
def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded. I will do this by DAS for now to
break the monstrous calls down into smaller chunks.
"""
# Grab all the Dataset-Algo combindations
dasList = self.dbsUtil.findUploadableDAS()
if len(dasList) < 1:
# Then there's nothing to do
return []
readyBlocks = []
for dasInfo in dasList:
dasID = dasInfo['DAS_ID']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(das=dasID)
except WMException:
raise
except Exception, ex:
msg = "Unhandled exception while loading uploadable files for DAS.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DAS being loaded: %s\n" % dasID)
raise DBSUploadException(msg)
# Get the blocks
if not dasID in self.dasCache.keys():
# Then we have a new DAS
# Add it
self.dasCache[dasID] = {}
dasBlocks = self.dasCache.get(dasID)
# Sort the files and blocks by location
fileDict = sortListByKey(input=loadedFiles, key='locations')
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if len(files) < 1:
# Nothing to do here
continue
dasBlocks = self.dasCache[dasID].get(location, [])
if len(dasBlocks) > 0:
# Load from cache
currentBlock = self.blockCache.get(dasBlocks[0])
else:
blockname = '%s#%s' % (files[0]['datasetPath'], makeUUID())
currentBlock = DBSBlock(name=blockname,
location=location,
das=dasID)
# Add the era info
currentBlock.setAcquisitionEra(
era=dasInfo['AcquisitionEra'])
currentBlock.setProcessingVer(
procVer=dasInfo['ProcessingVer'])
self.addNewBlock(block=currentBlock)
dasBlocks.append(currentBlock.getName())
for newFile in files:
if not newFile.get('block', 1) == None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile=newFile,
block=currentBlock):
# Then we have to close the block and get a new one
currentBlock.status = 'Pending'
readyBlocks.append(currentBlock)
dasBlocks.remove(currentBlock.getName())
currentBlock = self.getBlock(newFile=newFile,
dasBlocks=dasBlocks,
location=location,
das=dasID)
currentBlock.setAcquisitionEra(
era=dasInfo['AcquisitionEra'])
currentBlock.setProcessingVer(
procVer=dasInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(dbsFile=newFile)
self.filesToUpdate.append({
'filelfn': newFile['lfn'],
'block': currentBlock.getName()
})
# Done with the location
readyBlocks.append(currentBlock)
