def __init__(self): """ __init__ Init the DB """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.bufferFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") config = self.getConfig() self.dbsUploader = DBSUploadPoller(config=config) return
def preInitialization(self): print("DBS3Upload.preInitialization") # Add event loop to worker manager myThread = threading.currentThread() pollInterval = self.config.DBS3Upload.pollInterval logging.info("Setting poll interval to %s seconds" % pollInterval) myThread.workerThreadManager.addWorker(DBSUploadPoller(self.config), pollInterval) return
def testBasicUpload(self): """ _testBasicUpload_ Verify that we can successfully upload to DBS3. Also verify that the uploader correctly handles files parentage when uploading. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) # The algorithm needs to be run twice. On the first iteration it will # create all the blocks and upload one. On the second iteration it will # timeout and upload the second block. dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) # Verify the files made it into DBS3. self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Inject some more parent files and some child files into DBSBuffer. # Run the uploader twice, only the parent files should be added to DBS3. (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader another two times to upload the child files. Verify # that the child files were uploaded. dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return
def testDualUpload(self): """ _testDualUpload_ Verify that the dual upload mode works correctly. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) allFiles = parentFiles + moreParentFiles allBlocks = [] for i in range(4): DBSBufferDataset(parentFiles[0]["datasetPath"]).create() blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "Open" dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in allFiles[i * 5 : (i * 5) + 5]: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) if i < 2: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUtil.updateFileStatus([dbsBlock], "InDBS") allBlocks.append(dbsBlock) DBSBufferDataset(childFiles[0]["datasetPath"]).create() blockName = childFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "InDBS" dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in childFiles: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) dbsUtil.updateFileStatus([dbsBlock], "InDBS") dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Change the status of the rest of the parent blocks so we can upload # them and the children. for dbsBlock in allBlocks: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader one more time to upload the children. dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return
def testA_basicFunction(self): """ _basicFunction_ See if I can make the damn thing work. """ myThread = threading.currentThread() config = self.getConfig() from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller dbsUploader = DBSUploadPoller(config=config) dbsUtil = DBSBufferUtil() from dbs.apis.dbsClient import DbsApi dbsApi = DbsApi(url=config.DBSUpload.dbsUrl) # This should do nothing # Just making sure we don't crash try: dbsUploader.algorithm() except: dbsUploader.close() raise name = "ThisIsATest%s" % (int(time.time())) tier = "RECO" nFiles = 12 name = name.replace('-', '_') name = '%s-v0' % name files = self.getFiles(name=name, tier=tier, nFiles=nFiles) datasetPath = "/Cosmics/%s/%s" % (name, tier) try: dbsUploader.algorithm() except: dbsUploader.close() raise time.sleep(5) # Now look in DBS try: result = dbsApi.listDatasets(dataset=datasetPath, detail=True, dataset_access_type='PRODUCTION') self.assertEqual(len(result), 1) self.assertEqual(result[0]['data_tier_name'], 'RECO') self.assertEqual(result[0]['processing_version'], 0) self.assertEqual(result[0]['acquisition_era_name'], name.split('-')[0]) result = dbsApi.listFiles(dataset=datasetPath) self.assertEqual(len(result), 11) except: dbsUploader.close() raise # All the blocks except for the last one should # now be there result = myThread.dbi.processData( "SELECT id FROM dbsbuffer_block")[0].fetchall() self.assertEqual(len(result), 12) # The last block should still be open self.assertEqual(len(dbsUtil.findOpenBlocks()), 1) try: dbsUploader.algorithm() except: raise finally: dbsUploader.close() # All files should now be available result = dbsApi.listFiles(dataset=datasetPath) self.assertEqual(len(result), 12) # The last block should now be closed self.assertEqual(len(dbsUtil.findOpenBlocks()), 0) result = myThread.dbi.processData( "SELECT status FROM dbsbuffer_block")[0].fetchall() for res in result: self.assertEqual(res.values()[0], 'InDBS') return