def sortBlocks(self):
"""
_sortBlocks_
Find new files to upload, sort them into blocks
Save the blocks in DBSBuffer
"""
myThread = threading.currentThread()
# Grab all the Dataset-Algo combindations
dasList = self.uploadToDBS.findUploadableDAS()
logging.debug("Recovered %i DAS to upload" % len(dasList))
for dasInfo in dasList:
# Go one DAS at a time
dasID = dasInfo['DAS_ID']
logging.info("Processing DAS %i" % dasID)
# Initial values
readyBlocks = []
fileLFNs = []
# Get the dataset-algo information
#algo = createAlgoFromInfo(info = dasInfo)
dataset = createDatasetFromInfo(info = dasInfo)
# Get the files for the DAS
files = self.uploadToDBS.findUploadableFilesByDAS(das = dasID)
if len(files) < 1:
# Then we have no files for this DAS
logging.debug("DAS %i has no available files. Continuing." % dasID)
continue
# Load the blocks for the DAS
blocks = self.uploadToDBS.loadBlocksByDAS(das = dasID)
logging.debug("Retrieved %i files and %i blocks from DB." % (len(files), len(blocks)))
# Sort the files and blocks by location
locationDict = sortListByKey(files, 'locations')
blockDict = sortListByKey(blocks, 'location')
logging.debug("Active DAS file locations: %s" % locationDict.keys())
logging.debug("Active Block file locations: %s" % blockDict.keys())
try:
# Sort files that are already in blocks
# back into those blocks
# pass by reference
blockDict, locationDict = preassignBlocks(files = locationDict, blocks = blockDict)
# Now go over all the files
for location in locationDict.keys():
# Split files into blocks
locFiles = locationDict.get(location, [])
locBlocks = blockDict.get(location, [])
locBlocks = self.splitFilesIntoBlocks(files = locFiles,
blocks = locBlocks,
dataset = dataset,
location = location)
readyBlocks.extend(locBlocks)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while sorting files into blocks for DAS %i\n" % dasID
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(6, msg = msg)
logging.debug("BlockDictionary: %s" % blockDict)
logging.debug("FileDictionary: %s" % locationDict)
raise DBSUploadPollerException(msg)
# At this point, all blocks should be in readyBlocks
# STEP TWO: Commit blocks to DBSBuffer
fileLFNs = self.createBlocksInDBSBuffer(readyBlocks = readyBlocks)
# Now we should have all the blocks in DBSBuffer
# Time to set the status of the files
lfnList = [x['lfn'] for x in files]
self.setStatus.execute(lfns = lfnList, status = "READY",
conn = myThread.transaction.conn,
transaction = myThread.transaction)
# All files that were in NOTUPLOADED
# And had uploaded parents
# Should now be in assigned to blocks in DBSBuffer, and in the READY status
return
def sortBlocks(self):
"""
_sortBlocks_
Find new files to upload, sort them into blocks
Save the blocks in DBSBuffer
"""
myThread = threading.currentThread()
# Grab all the Dataset-Algo combindations
dasList = self.uploadToDBS.findUploadableDAS()
logging.debug("Recovered %i DAS to upload" % len(dasList))
for dasInfo in dasList:
# Go one DAS at a time
dasID = dasInfo['DAS_ID']
logging.info("Processing DAS %i" % dasID)
# Initial values
readyBlocks = []
fileLFNs = []
# Get the dataset-algo information
#algo = createAlgoFromInfo(info = dasInfo)
dataset = createDatasetFromInfo(info=dasInfo)
# Get the files for the DAS
files = self.uploadToDBS.findUploadableFilesByDAS(das=dasID)
if len(files) < 1:
# Then we have no files for this DAS
logging.debug("DAS %i has no available files. Continuing." %
dasID)
continue
# Load the blocks for the DAS
blocks = self.uploadToDBS.loadBlocksByDAS(das=dasID)
logging.debug("Retrieved %i files and %i blocks from DB." %
(len(files), len(blocks)))
# Sort the files and blocks by location
locationDict = sortListByKey(input=files, key='locations')
blockDict = sortListByKey(input=blocks, key='location')
logging.debug("Active DAS file locations: %s" %
locationDict.keys())
logging.debug("Active Block file locations: %s" % blockDict.keys())
try:
# Sort files that are already in blocks
# back into those blocks
# pass by reference
blockDict, locationDict = preassignBlocks(files=locationDict,
blocks=blockDict)
# Now go over all the files
for location in locationDict.keys():
# Split files into blocks
locFiles = locationDict.get(location, [])
locBlocks = blockDict.get(location, [])
locBlocks = self.splitFilesIntoBlocks(files=locFiles,
blocks=locBlocks,
dataset=dataset,
location=location)
readyBlocks.extend(locBlocks)
except WMException:
raise
except Exception, ex:
msg = "Unhandled exception while sorting files into blocks for DAS %i\n" % dasID
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
self.sendAlert(6, msg=msg)
logging.debug("BlockDictionary: %s" % blockDict)
logging.debug("FileDictionary: %s" % locationDict)
raise DBSUploadPollerException(msg)
# At this point, all blocks should be in readyBlocks
# STEP TWO: Commit blocks to DBSBuffer
fileLFNs = self.createBlocksInDBSBuffer(readyBlocks=readyBlocks)
# Now we should have all the blocks in DBSBuffer
# Time to set the status of the files
lfnList = [x['lfn'] for x in files]
self.setStatus.execute(lfns=lfnList,
status="READY",
conn=myThread.transaction.conn,
transaction=myThread.transaction)
def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded. I will do this by DatasetPath
to break the monstrous calls down into smaller chunks.
"""
dspList = self.dbsUtil.findUploadableDAS()
readyBlocks = []
for dspInfo in dspList:
datasetpath = dspInfo['DatasetPath']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(
datasetpath=datasetpath)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading uploadable files for DatasetPath.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DatasetPath being loaded: %s\n", datasetpath)
raise DBSUploadException(msg)
# Sort the files and blocks by location
fileDict = sortListByKey(loadedFiles, 'locations')
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if files:
currentBlock = self.getBlock(files[0], location, True)
currentBlock.setAcquisitionEra(
era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(
procVer=dspInfo['ProcessingVer'])
for newFile in files:
if not newFile.get('block', 1) is None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile=newFile,
block=currentBlock):
# Then we have to close the block and get a new one
currentBlock.setPendingAndCloseBlock()
readyBlocks.append(currentBlock)
currentBlock = self.getBlock(newFile=newFile,
location=location)
currentBlock.setAcquisitionEra(
era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(
procVer=dspInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(newFile, self.datasetType,
self.primaryDatasetType)
self.filesToUpdate.append({
'filelfn':
newFile['lfn'],
'block':
currentBlock.getName()
})
# Done with the location
readyBlocks.append(currentBlock)
for block in readyBlocks:
self.blockCache[block.getName()] = block
return
def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded. I will do this by DatasetPath
to break the monstrous calls down into smaller chunks.
"""
dspList = self.dbsUtil.findUploadableDAS()
readyBlocks = []
for dspInfo in dspList:
datasetpath = dspInfo['DatasetPath']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(datasetpath=datasetpath)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading uploadable files for DatasetPath.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DatasetPath being loaded: %s\n", datasetpath)
raise DBSUploadException(msg)
# Sort the files and blocks by location
fileDict = sortListByKey(loadedFiles, 'locations')
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if len(files) > 0:
currentBlock = self.getBlock(files[0], location, True)
currentBlock.setAcquisitionEra(era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(procVer=dspInfo['ProcessingVer'])
for newFile in files:
if not newFile.get('block', 1) is None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile=newFile,
block=currentBlock):
# Then we have to close the block and get a new one
currentBlock.setPendingAndCloseBlock()
readyBlocks.append(currentBlock)
currentBlock = self.getBlock(newFile=newFile,
location=location)
currentBlock.setAcquisitionEra(era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(procVer=dspInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(newFile, self.datasetType, self.primaryDatasetType)
self.filesToUpdate.append({'filelfn': newFile['lfn'],
'block': currentBlock.getName()})
# Done with the location
readyBlocks.append(currentBlock)
for block in readyBlocks:
self.blockCache[block.getName()] = block
return
