except Exception, ex: msg = "Unhandled exception while loading blocks.\n" msg += str(ex) logging.error(msg) logging.debug("Blocks to load: %s\n" % blocksToLoad) raise DBSUploadException(msg) for blockInfo in loadedBlocks: das = blockInfo['DatasetAlgo'] loc = blockInfo['origin_site_name'] workflow = blockInfo['workflow'] block = DBSBlock(name=blockInfo['block_name'], location=loc, das=das, workflow=workflow) block.FillFromDBSBuffer(blockInfo) blockname = block.getName() # Now we have to load files... try: files = self.dbsUtil.loadFilesByBlock(blockname=blockname) logging.info("Have %i files for block %s" % (len(files), blockname)) except WMException: raise except Exception, ex: msg = "Unhandled exception while loading files for existing blocks.\n" msg += str(ex) logging.error(msg) logging.debug("Blocks being loaded: %s\n" % blockname) raise DBSUploadException(msg)
def loadBlocks(self): """ _loadBlocks_ Find all blocks; make sure they're in the cache """ openBlocks = self.dbsUtil.findOpenBlocks(self.dbs3UploadOnly) logging.info("These are the openblocks: %s" % openBlocks) # Load them if we don't have them blocksToLoad = [] for block in openBlocks: if not block['blockname'] in self.blockCache.keys(): blocksToLoad.append(block['blockname']) # Now load the blocks try: loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad, self.dbs3UploadOnly) logging.info("Loaded blocks: %s" % loadedBlocks) except WMException: raise except Exception as ex: msg = "Unhandled exception while loading blocks.\n" msg += str(ex) logging.error(msg) logging.debug("Blocks to load: %s\n" % blocksToLoad) raise DBSUploadException(msg) for blockInfo in loadedBlocks: das = blockInfo['DatasetAlgo'] loc = blockInfo['origin_site_name'] workflow = blockInfo['workflow'] block = DBSBlock(name = blockInfo['block_name'], location = loc, das = das, workflow = workflow) block.FillFromDBSBuffer(blockInfo) blockname = block.getName() # Now we have to load files... try: files = self.dbsUtil.loadFilesByBlock(blockname = blockname) logging.info("Have %i files for block %s" % (len(files), blockname)) except WMException: raise except Exception as ex: msg = "Unhandled exception while loading files for existing blocks.\n" msg += str(ex) logging.error(msg) logging.debug("Blocks being loaded: %s\n" % blockname) raise DBSUploadException(msg) # Add the loaded files to the block for file in files: block.addFile(file, self.datasetType, self.primaryDatasetType) # Add to the cache self.addNewBlock(block = block) # All blocks should now be loaded and present # in both the block cache (which has all the info) # and the dasCache (which is a list of name pointers # to the keys in the block cache). return