def _createDatasetSubscriptionsInDBSBuffer(self): """ _createDatasetSubscriptionsInDBSBuffer_ Insert the subscriptions defined in the workload for the output datasets with the different options. """ subInfo = self.wmSpec.getSubscriptionInformation() for dataset in subInfo: dbsDataset = DBSBufferDataset(path = dataset) dbsDataset.create() dbsDataset.addSubscription(subInfo[dataset]) return
def _createDatasetSubscriptionsInDBSBuffer(self): """ _createDatasetSubscriptionsInDBSBuffer_ Insert the subscriptions defined in the workload for the output datasets with the different options. """ subInfo = self.wmSpec.getSubscriptionInformation() for dataset in subInfo: dbsDataset = DBSBufferDataset(path=dataset) dbsDataset.create() dbsDataset.addSubscription(subInfo[dataset]) return
def insertDatasetAlgo(self): """ _insertDatasetAlgo_ Insert the dataset and algorithm for this file into the DBS Buffer. """ newAlgoAction = self.daofactory(classname="NewAlgo") assocAction = self.daofactory(classname="AlgoDatasetAssoc") existingTransaction = self.beginTransaction() newAlgoAction.execute(appName=self["appName"], appVer=self["appVer"], appFam=self["appFam"], psetHash=self["psetHash"], configContent=self["configContent"], conn=self.getDBConn(), transaction=True) dbsbufferDataset = DBSBufferDataset( self["datasetPath"], processingVer=self['processingVer'], acquisitionEra=self['acquisitionEra'], validStatus=self['validStatus'], globalTag=self.get('globalTag', None), parent=self['datasetParent'], prep_id=self['prep_id']) if dbsbufferDataset.exists(): dbsbufferDataset.updateDataset() else: dbsbufferDataset.create() assocID = assocAction.execute(appName=self["appName"], appVer=self["appVer"], appFam=self["appFam"], psetHash=self["psetHash"], datasetPath=self["datasetPath"], conn=self.getDBConn(), transaction=True) self.commitTransaction(existingTransaction) return assocID
def insertDatasetAlgo(self): """ _insertDatasetAlgo_ Insert the dataset and algorithm for this file into the DBS Buffer. """ newAlgoAction = self.daofactory(classname="NewAlgo") assocAction = self.daofactory(classname="AlgoDatasetAssoc") existingTransaction = self.beginTransaction() newAlgoAction.execute(appName=self["appName"], appVer=self["appVer"], appFam=self["appFam"], psetHash=self["psetHash"], configContent=self["configContent"], conn=self.getDBConn(), transaction=True) dbsbufferDataset = DBSBufferDataset(self["datasetPath"], processingVer=self['processingVer'], acquisitionEra=self['acquisitionEra'], validStatus=self['validStatus'], globalTag=self.get('globalTag', None), parent=self['datasetParent'], prep_id=self['prep_id']) if dbsbufferDataset.exists(): dbsbufferDataset.updateDataset() else: dbsbufferDataset.create() assocID = assocAction.execute(appName=self["appName"], appVer=self["appVer"], appFam=self["appFam"], psetHash=self["psetHash"], datasetPath=self["datasetPath"], conn=self.getDBConn(), transaction=True) self.commitTransaction(existingTransaction) return assocID
def stuffDatabase(self, tier0Mode = False): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createBlock = uploadFactory(classname = "SetBlockStatus") self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname = "NewSubscription") datasetA = DBSBufferDataset(path = self.testDatasetA) datasetB = DBSBufferDataset(path = self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) if tier0Mode: # Override the settings workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"], nonCustodialSites = ["T3_CO_Uniandes"], priority = "Normal", custodialSubType = "Replica", autoApproveSites = ["T0_CH_CERN"], dataTier = "RECO") workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"], nonCustodialSites = [], autoApproveSites = [], priority = "High", custodialSubType = "Replica", dataTier = "RAW") insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return
def testDualUpload(self): """ _testDualUpload_ Verify that the dual upload mode works correctly. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) allFiles = parentFiles + moreParentFiles allBlocks = [] for i in range(4): DBSBufferDataset(parentFiles[0]["datasetPath"]).create() blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "Open" dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in allFiles[i * 5 : (i * 5) + 5]: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) if i < 2: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUtil.updateFileStatus([dbsBlock], "InDBS") allBlocks.append(dbsBlock) DBSBufferDataset(childFiles[0]["datasetPath"]).create() blockName = childFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "InDBS" dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in childFiles: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) dbsUtil.updateFileStatus([dbsBlock], "InDBS") dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Change the status of the rest of the parent blocks so we can upload # them and the children. for dbsBlock in allBlocks: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader one more time to upload the children. dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return
def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) datasetAction = uploadFactory(classname = "NewDataset") createAction = uploadFactory(classname = "CreateBlocks") datasetAction.execute(datasetPath = self.testDatasetA) datasetAction.execute(datasetPath = self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBlock(name = self.blockAName, location = "srm-cms.cern.ch", das = None, workflow = None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBlock(name = self.blockBName, location = "srm-cms.cern.ch", das = None, workflow = None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks = [newBlockA, newBlockB]) bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname = "NewSubscription") datasetA = DBSBufferDataset(path = self.testDatasetA) datasetB = DBSBufferDataset(path = self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return
def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) insertWorkflow = buffer3Factory(classname="InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) datasetAction = uploadFactory(classname="NewDataset") createAction = uploadFactory(classname="CreateBlocks") datasetAction.execute(datasetPath=self.testDatasetA) datasetAction.execute(datasetPath=self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBufferBlock(name=self.blockAName, location="srm-cms.cern.ch", datasetpath=None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBufferBlock(name=self.blockBName, location="srm-cms.cern.ch", datasetpath=None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks=[newBlockA, newBlockB]) bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname="DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname="NewSubscription") datasetA = DBSBufferDataset(path=self.testDatasetA) datasetB = DBSBufferDataset(path=self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return
def stuffDatabase(self, tier0Mode = False): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createBlock = uploadFactory(classname = "SetBlockStatus") self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname = "NewSubscription") datasetA = DBSBufferDataset(path = self.testDatasetA) datasetB = DBSBufferDataset(path = self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) if tier0Mode: # Override the settings workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"], nonCustodialSites = ["T3_CO_Uniandes"], priority = "Normal", custodialSubType = "Replica", autoApproveSites = ["T0_CH_CERN"], dataTier = "RECO") workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"], nonCustodialSites = [], autoApproveSites = [], priority = "High", custodialSubType = "Replica", dataTier = "RAW") insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return
def testBasic(self): """ _testBasic_ Test the basic functions of the DBSBufferDataset, create, load, exists and also the ability to add subscriptions. """ originalDataset = DBSBufferDataset(path='/bogus/bogus/go') originalDataset.create() myThread = threading.currentThread() result = myThread.dbi.processData( "SELECT id FROM dbsbuffer_dataset")[0].fetchall() self.assertEqual(originalDataset.exists(), result[0][0]) duplicateDataset = DBSBufferDataset(path='/bogus/bogus/go') duplicateDataset.create() self.assertEqual(originalDataset.exists(), duplicateDataset.exists()) result = myThread.dbi.processData( "SELECT COUNT(id) FROM dbsbuffer_dataset")[0].fetchall() self.assertEqual(result[0][0], 1) loadedDataset = DBSBufferDataset(path='/bogus/bogus/go') loadedDataset.load() self.assertEqual(loadedDataset.exists(), originalDataset.exists()) secondDataset = DBSBufferDataset( path='/BogusPrimary/Run2012Z-PromptReco-v1/RECO') secondDataset.create() workload = WMWorkloadHelper() workload.load( os.path.join( getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) secondDataset.addSubscription( workload.getSubscriptionInformation() ['/BogusPrimary/Run2012Z-PromptReco-v1/RECO']) secondDataset.addSubscription( workload.getSubscriptionInformation() ['/BogusPrimary/Run2012Z-PromptReco-v1/RECO']) self.assertEqual(len(secondDataset['subscriptions']), 3) result = myThread.dbi.processData( "SELECT COUNT(id) FROM dbsbuffer_dataset_subscription" )[0].fetchall() self.assertEqual(result[0][0], 3) return
def testBasic(self): """ _testBasic_ Test the basic functions of the DBSBufferDataset, create, load, exists and also the ability to add subscriptions. """ originalDataset = DBSBufferDataset(path = '/bogus/bogus/go') originalDataset.create() myThread = threading.currentThread() result = myThread.dbi.processData("SELECT id FROM dbsbuffer_dataset")[0].fetchall() self.assertEqual(originalDataset.exists(), result[0][0]) duplicateDataset = DBSBufferDataset(path = '/bogus/bogus/go') duplicateDataset.create() self.assertEqual(originalDataset.exists(), duplicateDataset.exists()) result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset")[0].fetchall() self.assertEqual(result[0][0], 1) loadedDataset = DBSBufferDataset(path = '/bogus/bogus/go') loadedDataset.load() self.assertEqual(loadedDataset.exists(), originalDataset.exists()) secondDataset = DBSBufferDataset(path = '/BogusPrimary/Run2012Z-PromptReco-v1/RECO') secondDataset.create() workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO']) secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO']) self.assertEqual(len(secondDataset['subscriptions']), 3) result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset_subscription")[0].fetchall() self.assertEqual(result[0][0], 3) return