def loadBlocks(self): """ _loadBlocks_ Find all blocks; make sure they're in the cache """ openBlocks = self.dbsUtil.findOpenBlocks() logging.info("Found %d open blocks.", len(openBlocks)) logging.debug("These are the openblocks: %s", openBlocks) # Load them if we don't have them blocksToLoad = [] for block in openBlocks: if not block['blockname'] in self.blockCache.keys(): blocksToLoad.append(block['blockname']) # Now load the blocks try: loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad) logging.info("Loaded %d blocks.", len(loadedBlocks)) except WMException: raise except Exception as ex: msg = "Unhandled exception while loading blocks.\n" msg += str(ex) logging.error(msg) logging.debug("Blocks to load: %s\n", blocksToLoad) raise DBSUploadException(msg) for blockInfo in loadedBlocks: block = DBSBufferBlock(name=blockInfo['block_name'], location=blockInfo['origin_site_name'], datasetpath=blockInfo['datasetpath']) block.FillFromDBSBuffer(blockInfo) blockname = block.getName() # Now we have to load files... try: files = self.dbsUtil.loadFilesByBlock(blockname=blockname) logging.info("Have %i files for block %s", len(files), blockname) except WMException: raise except Exception as ex: msg = "Unhandled exception while loading files for existing blocks.\n" msg += str(ex) logging.error(msg) logging.debug("Blocks being loaded: %s\n", blockname) raise DBSUploadException(msg) # Add the loaded files to the block for f in files: block.addFile(f, self.datasetType, self.primaryDatasetType) # Add to the cache self.blockCache[blockInfo['block_name']] = block return
def testDualUpload(self): """ _testDualUpload_ Verify that the dual upload mode works correctly. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) allFiles = parentFiles + moreParentFiles allBlocks = [] for i in range(4): DBSBufferDataset(parentFiles[0]["datasetPath"]).create() blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "Open" dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in allFiles[i * 5 : (i * 5) + 5]: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) if i < 2: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUtil.updateFileStatus([dbsBlock], "InDBS") allBlocks.append(dbsBlock) DBSBufferDataset(childFiles[0]["datasetPath"]).create() blockName = childFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "InDBS" dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in childFiles: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) dbsUtil.updateFileStatus([dbsBlock], "InDBS") dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Change the status of the rest of the parent blocks so we can upload # them and the children. for dbsBlock in allBlocks: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader one more time to upload the children. dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return
def testDualUpload(self): """ _testDualUpload_ Verify that the dual upload mode works correctly. """ self.dbsApi = DbsApi(url=self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config=config) dbsUtil = DBSBufferUtil() # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) (moreParentFiles, childFiles) = self.createFilesWithChildren(parentFiles, acqEra) allFiles = parentFiles + moreParentFiles allBlocks = [] for i in range(4): DBSBufferDataset(parentFiles[0]["datasetPath"]).create() blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location="malpaquet", datasetpath=None) dbsBlock.status = "Open" dbsBlock.setDataset(parentFiles[0]["datasetPath"], "data", "VALID") dbsUtil.createBlocks([dbsBlock]) for file in allFiles[i * 5 : (i * 5) + 5]: dbsBlock.addFile(file, "data", "VALID") dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) if i < 2: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUtil.updateFileStatus([dbsBlock], "InDBS") allBlocks.append(dbsBlock) DBSBufferDataset(childFiles[0]["datasetPath"]).create() blockName = childFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location="malpaquet", datasetpath=None) dbsBlock.status = "InDBS" dbsBlock.setDataset(childFiles[0]["datasetPath"], "data", "VALID") dbsUtil.createBlocks([dbsBlock]) for file in childFiles: dbsBlock.addFile(file, "data", "VALID") dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) dbsUtil.updateFileStatus([dbsBlock], "InDBS") dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Change the status of the rest of the parent blocks so we can upload # them and the children. for dbsBlock in allBlocks: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader one more time to upload the children. dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return