except Exception, ex:
msg = "Unhandled exception while loading blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks to load: %s\n" % blocksToLoad)
raise DBSUploadException(msg)
for blockInfo in loadedBlocks:
das = blockInfo['DatasetAlgo']
loc = blockInfo['origin_site_name']
workflow = blockInfo['workflow']
block = DBSBlock(name=blockInfo['block_name'],
location=loc,
das=das,
workflow=workflow)
block.FillFromDBSBuffer(blockInfo)
blockname = block.getName()
# Now we have to load files...
try:
files = self.dbsUtil.loadFilesByBlock(blockname=blockname)
logging.info("Have %i files for block %s" %
(len(files), blockname))
except WMException:
raise
except Exception, ex:
msg = "Unhandled exception while loading files for existing blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks being loaded: %s\n" % blockname)
raise DBSUploadException(msg)
def loadBlocks(self):
"""
_loadBlocks_
Find all blocks; make sure they're in the cache
"""
openBlocks = self.dbsUtil.findOpenBlocks(self.dbs3UploadOnly)
logging.info("These are the openblocks: %s" % openBlocks)
# Load them if we don't have them
blocksToLoad = []
for block in openBlocks:
if not block['blockname'] in self.blockCache.keys():
blocksToLoad.append(block['blockname'])
# Now load the blocks
try:
loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad, self.dbs3UploadOnly)
logging.info("Loaded blocks: %s" % loadedBlocks)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks to load: %s\n" % blocksToLoad)
raise DBSUploadException(msg)
for blockInfo in loadedBlocks:
das = blockInfo['DatasetAlgo']
loc = blockInfo['origin_site_name']
workflow = blockInfo['workflow']
block = DBSBlock(name = blockInfo['block_name'],
location = loc, das = das, workflow = workflow)
block.FillFromDBSBuffer(blockInfo)
blockname = block.getName()
# Now we have to load files...
try:
files = self.dbsUtil.loadFilesByBlock(blockname = blockname)
logging.info("Have %i files for block %s" % (len(files), blockname))
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading files for existing blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks being loaded: %s\n" % blockname)
raise DBSUploadException(msg)
# Add the loaded files to the block
for file in files:
block.addFile(file, self.datasetType, self.primaryDatasetType)
# Add to the cache
self.addNewBlock(block = block)
# All blocks should now be loaded and present
# in both the block cache (which has all the info)
# and the dasCache (which is a list of name pointers
# to the keys in the block cache).
return