class DBSUploadTest(unittest.TestCase): """ TestCase for DBSUpload module """ def setUp(self): """ _setUp_ setUp function for unittest """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"], useDefault = False) self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) self.buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation") locationAction.execute(siteName = "se1.cern.ch") locationAction.execute(siteName = "se1.fnal.gov") locationAction.execute(siteName = "malpaquet") self.dbsUrl = "https://*****:*****@attr("integration") def testBasicUpload(self): """ _testBasicUpload_ Verify that we can successfully upload to DBS3. Also verify that the uploader correctly handles files parentage when uploading. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) # The algorithm needs to be run twice. On the first iteration it will # create all the blocks and upload one. On the second iteration it will # timeout and upload the second block. dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) # Verify the files made it into DBS3. self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Inject some more parent files and some child files into DBSBuffer. # Run the uploader twice, only the parent files should be added to DBS3. (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader another two times to upload the child files. Verify # that the child files were uploaded. dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return @attr("integration") def testDualUpload(self): """ _testDualUpload_ Verify that the dual upload mode works correctly. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig(dbs3UploadOnly = True) dbsUploader = DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) allFiles = parentFiles + moreParentFiles allBlocks = [] for i in range(4): blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBlock(blockName, "malpaquet", 1) dbsBlock.status = "Open" dbsUtil.createBlocks([dbsBlock]) for file in allFiles[i * 5 : (i * 5) + 5]: dbsBlock.addFile(file) dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) if i < 2: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUtil.updateFileStatus([dbsBlock], "InDBS") allBlocks.append(dbsBlock) blockName = childFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBlock(blockName, "malpaquet", 1) dbsBlock.status = "InDBS" dbsUtil.createBlocks([dbsBlock]) for file in childFiles: dbsBlock.addFile(file) dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) dbsUtil.updateFileStatus([dbsBlock], "InDBS") dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Change the status of the rest of the parent blocks so we can upload # them and the children. for dbsBlock in allBlocks: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader one more time to upload the children. dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return def testCloseSettingsPerWorkflow(self): """ _testCloseSettingsPerWorkflow_ Test the block closing mechanics in the DBS3 uploader, this uses a fake dbs api to avoid reliance on external services. """ # Signal trapExit that we are a friend os.environ["DONT_TRAP_EXIT"] = "True" try: # Monkey patch the imports of DbsApi from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller MockDBSUploadPoller.DbsApi = MockDbsApi # Set the poller and the dbsUtil for verification myThread = threading.currentThread() (_, dbsFilePath) = mkstemp(dir = self.testDir) self.dbsUrl = dbsFilePath config = self.getConfig() dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test is event based limits and timeout with no new files. # Set the files and workflow acqEra = "TropicalSeason%s" % (int(time.time())) workflowName = 'TestWorkload%s' % (int(time.time())) taskPath = '/%s/TestProcessing' % workflowName self.injectWorkflow(workflowName, taskPath, MaxWaitTime = 2, MaxFiles = 100, MaxEvents = 150) self.createParentFiles(acqEra, nFiles = 20, workflowName = workflowName, taskPath = taskPath) # The algorithm needs to be run twice. On the first iteration it will # create all the blocks and upload one with less than 150 events. # On the second iteration the second block is uploaded. dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 1) globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall() notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall() self.assertEqual(len(globalFiles), 14) self.assertEqual(len(notUploadedFiles), 6) # Check the fake DBS for data fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 2) for block in fakeDBSInfo: self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['file_count'], 7) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) time.sleep(3) dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 0) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 3) for block in fakeDBSInfo: if block['block']['file_count'] != 6: self.assertEqual(block['block']['file_count'], 7) self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) # Now check the limit by size and timeout with new files acqEra = "TropicalSeason%s" % (int(time.time())) workflowName = 'TestWorkload%s' % (int(time.time())) taskPath = '/%s/TestProcessing' % workflowName self.injectWorkflow(workflowName, taskPath, MaxWaitTime = 2, MaxFiles = 5, MaxEvents = 200000000) self.createParentFiles(acqEra, nFiles = 16, workflowName = workflowName, taskPath = taskPath) dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 1) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 6) for block in fakeDBSInfo: if acqEra in block['block']['block_name']: self.assertEqual(block['block']['file_count'], 5) self.assertTrue('block_events' not in block['block']) self.assertTrue('close_settings' not in block) self.assertEqual(block['block']['open_for_writing'], 0) # Put more files, they will go into the same block and then it will be closed # after timeout time.sleep(3) self.createParentFiles(acqEra, nFiles = 3, workflowName = workflowName, taskPath = taskPath) dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 0) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 7) for block in fakeDBSInfo: if acqEra in block['block']['block_name']: if block['block']['file_count'] < 5: self.assertEqual(block['block']['file_count'], 4) else: self.assertEqual(block['block']['file_count'], 5) self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) # Finally test size limits acqEra = "TropicalSeason%s" % (int(time.time())) workflowName = 'TestWorkload%s' % (int(time.time())) taskPath = '/%s/TestProcessing' % workflowName self.injectWorkflow(workflowName, taskPath, MaxWaitTime = 1, MaxFiles = 500, MaxEvents = 200000000, MaxSize = 2048) self.createParentFiles(acqEra, nFiles = 7, workflowName = workflowName, taskPath = taskPath) dbsUploader.algorithm() dbsUploader.checkBlocks() time.sleep(2) dbsUploader.algorithm() dbsUploader.checkBlocks() self.assertEqual(len(openBlocks), 0) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 11) for block in fakeDBSInfo: if acqEra in block['block']['block_name']: if block['block']['file_count'] != 1: self.assertEqual(block['block']['block_size'], 2048) self.assertEqual(block['block']['file_count'], 2) self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) except: self.fail("We failed at some point in the test") finally: # We don't trust anyone else with _exit del os.environ["DONT_TRAP_EXIT"] return
class ThreadPoolTest(unittest.TestCase): """ _ThreadPool_t_ Unit tests for threadpool """ _nrOfThreads = 10 _nrOfPools = 5 def setUp(self): "make a logger instance and create tables" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() def tearDown(self): """ Deletion of database """ # FIXME: this might not work if your not using socket. self.testInit.clearDatabase() def testA(self): """ __testSubscribe__ Test subscription of a component. """ raise nose.SkipTest myThread = threading.currentThread() # create a 'fake' component that contains a arg dictionary. component = Dummy() # load default parameters. config = self.testInit.getConfiguration() # normally assigned by the harness of the test component. config.Agent.componentName = "TestComponent" component.config = config threadPools = [] for i in xrange(0, ThreadPoolTest._nrOfPools): threadPool = ThreadPool("WMCore.ThreadPool.ThreadSlave", \ component, 'MyPool_'+str(i), ThreadPoolTest._nrOfThreads) threadPools.append(threadPool) # this is how you would use the threadpool. The threadpool retrieves # events/payloads from the message service. If a thread is available # it is dispatched, otherwise it is stored in the trheadpool. # make the number of tasks bigger than number of threads to tesT # the persistent queue. for i in xrange(0, ThreadPoolTest._nrOfThreads * 10): event = 'eventNr_' + str(i) payload = 'payloadNr_' + str(i) # normally you would have different events per threadpool and # even different objects per pool. the payload part will be # pickled into the database enabling flexibility in passing # information. for j in xrange(0, ThreadPoolTest._nrOfPools): threadPools[j].enqueue(event, \ {'event' : event, 'payload' : payload}) # this commit you want to be in the agent harness, so the message is # actual removed from the msgService. we can do this as the threadpool # acts as a dispatcher and is a shortlived action: dispatch to thread # or queue and tell agent harness it is finished. finished = False timeout = 60 # secs currenttime = 0 while not finished: print('waiting for threads to finishs. Work left:') for j in xrange(0, ThreadPoolTest._nrOfPools): print('pool_' + str(j) + ':' + str(threadPools[j].callQueue)) time.sleep(1) finished = True currenttime += 1 if (timeout == currenttime): raise RuntimeError for j in xrange(0, ThreadPoolTest._nrOfPools): if (len(threadPools[j].resultsQueue) < ThreadPoolTest._nrOfThreads * 10): finished = False break # check if the tables are really empty and all messages # have been processed. for j in xrange(0, ThreadPoolTest._nrOfPools): assert len(threadPools[j].resultsQueue) == \ ThreadPoolTest._nrOfThreads*10 myThread.transaction.begin() for j in xrange(0, ThreadPoolTest._nrOfPools): self.assertEqual(threadPools[j].countMessages(), 0) myThread.transaction.commit()
class ThreadPoolTest(unittest.TestCase): """ _ThreadPool_t_ Unit tests for threadpool """ _nrOfThreads = 10 _nrOfPools = 5 def setUp(self): "make a logger instance and create tables" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() def tearDown(self): """ Deletion of database """ # FIXME: this might not work if your not using socket. self.testInit.clearDatabase() def testA(self): """ __testSubscribe__ Test subscription of a component. """ raise nose.SkipTest myThread = threading.currentThread() # create a 'fake' component that contains a arg dictionary. component = Dummy() # load default parameters. config = self.testInit.getConfiguration() # normally assigned by the harness of the test component. config.Agent.componentName = "TestComponent" component.config = config threadPools = [] for i in xrange(0, ThreadPoolTest._nrOfPools): threadPool = ThreadPool("WMCore.ThreadPool.ThreadSlave", \ component, 'MyPool_'+str(i), ThreadPoolTest._nrOfThreads) threadPools.append(threadPool) # this is how you would use the threadpool. The threadpool retrieves # events/payloads from the message service. If a thread is available # it is dispatched, otherwise it is stored in the trheadpool. # make the number of tasks bigger than number of threads to tesT # the persistent queue. for i in xrange(0, ThreadPoolTest._nrOfThreads*10): event = 'eventNr_'+str(i) payload = 'payloadNr_'+str(i) # normally you would have different events per threadpool and # even different objects per pool. the payload part will be # pickled into the database enabling flexibility in passing # information. for j in xrange(0, ThreadPoolTest._nrOfPools): threadPools[j].enqueue(event, \ {'event' : event, 'payload' : payload}) # this commit you want to be in the agent harness, so the message is # actual removed from the msgService. we can do this as the threadpool # acts as a dispatcher and is a shortlived action: dispatch to thread # or queue and tell agent harness it is finished. finished = False timeout = 60 # secs currenttime = 0 while not finished: print('waiting for threads to finishs. Work left:') for j in xrange(0, ThreadPoolTest._nrOfPools): print('pool_'+str(j)+ ':' + str(threadPools[j].callQueue)) time.sleep(1) finished = True currenttime += 1 if (timeout == currenttime): raise RuntimeError for j in xrange(0, ThreadPoolTest._nrOfPools): if (len(threadPools[j].resultsQueue) < ThreadPoolTest._nrOfThreads*10): finished = False break # check if the tables are really empty and all messages # have been processed. for j in xrange(0, ThreadPoolTest._nrOfPools): assert len(threadPools[j].resultsQueue) == \ ThreadPoolTest._nrOfThreads*10 myThread.transaction.begin() for j in xrange(0, ThreadPoolTest._nrOfPools): self.assertEqual( threadPools[j].countMessages() , 0 ) myThread.transaction.commit()
class scaleTestFiller: """ _scaleTestFiller_ Initializes the DB and the DBSUploader On __call__() it creates data and uploads it. """ def __init__(self): """ __init__ Init the DB """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase = True) self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"], useDefault = False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation") locationAction.execute(siteName = "se1.cern.ch") locationAction.execute(siteName = "se1.fnal.gov") locationAction.execute(siteName = "malpaquet") config = self.getConfig() self.dbsUploader = DBSUploadPoller(config = config) return def __call__(self): """ __call__ Generate some random data """ # Generate somewhere between one and a thousand files name = "ThisIsATest_%s" % (makeUUID()) nFiles = random.randint(10, 2000) name = name.replace('-', '_') name = '%s-v0' % name files = self.getFiles(name = name, nFiles = nFiles) print("Inserting %i files for dataset %s" % (nFiles * 2, name)) try: self.dbsUploader.algorithm() except: self.dbsUploader.close() raise # Repeat just to make sure try: self.dbsUploader.algorithm() except: self.dbsUploader.close() raise return def getConfig(self): """ _getConfig_ This creates the actual config file used by the component """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBS3Upload") config.DBS3Upload.pollInterval = 10 config.DBS3Upload.logLevel = 'DEBUG' config.DBS3Upload.DBSBlockMaxFiles = 500 config.DBS3Upload.DBSBlockMaxTime = 600 config.DBS3Upload.DBSBlockMaxSize = 999999999999 config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBS3Upload' config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBS3Upload.nProcesses = 1 config.DBS3Upload.dbsWaitTime = 1 return config def getFiles(self, name, tier = 'RECO', nFiles = 12, site = "malpaquet", nLumis = 1): """ Create some quick dummy test files """ files = [] for f in range(nFiles): testFile = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size = 1024, events = 20, checksums = {'cksum': 1}) testFile.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) lumis = [] for i in range(nLumis): lumis.append((f * 100000) + i) testFile.addRun(Run( 1, *lumis)) testFile.setAcquisitionEra(name.split('-')[0]) testFile.setProcessingVer("0") testFile.setGlobalTag("Weird") testFile.create() testFile.setLocation(site) files.append(testFile) count = 0 for f in files: count += 1 testFileChild = DBSBufferFile(lfn = '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' %(name, site, count), size = 1024, events = 10, checksums = {'cksum': 1}) testFileChild.setAlgorithm(appName = name, appVer = "CMSSW_3_1_1", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileChild.setDatasetPath("/%s/%s_2/RECO" %(name, name)) testFileChild.addRun(Run( 1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([f['lfn']]) return files
class PhEDExInjectorPollerTest(unittest.TestCase): """ _PhEDExInjectorPollerTest_ Unit tests for the PhEDExInjector. Create some database inside DBSBuffer and then have the PhEDExInjector upload the data to PhEDEx. Pull the data back down and verify that everything is complete. """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/test" self.dbsURL = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="srm-cms.cern.ch") locationAction.execute(siteName="se.fnal.gov") self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/%s/PromptReco-v1/RECO" % makeUUID() self.testDatasetB = "/%s/CRUZET11-v1/RAW" % makeUUID() self.phedex = PhEDEx({"endpoint": self.phedexURL}, "json") return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() def stuffDatabase(self, spec="TestWorkload.pkl"): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the datbase. We'll inject files with the location set as an SE name as well as a PhEDEx node name as well. """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) insertWorkflow = buffer3Factory(classname="InsertWorkflow") insertWorkflow.execute("BogusRequest", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) datasetAction = uploadFactory(classname="NewDataset") createAction = uploadFactory(classname="CreateBlocks") datasetAction.execute(datasetPath=self.testDatasetA) datasetAction.execute(datasetPath=self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBlock(name=self.blockAName, location="srm-cms.cern.ch", das=None, workflow=None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBlock(name=self.blockBName, location="srm-cms.cern.ch", das=None, workflow=None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks=[newBlockA, newBlockB]) bufferFactory = DAOFactory(package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "LOCAL") fileStatus.execute(testFileB["lfn"], "LOCAL") fileStatus.execute(testFileC["lfn"], "LOCAL") fileStatus.execute(testFileD["lfn"], "LOCAL") fileStatus.execute(testFileE["lfn"], "LOCAL") associateWorkflow = buffer3Factory( classname="DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask") return def createConfig(self): """ _createConfig_ Create a config for the PhEDExInjector with paths to the test DBS and PhEDEx instances. """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeMSS = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 return config def retrieveReplicaInfoForBlock(self, blockName): """ _retrieveReplicaInfoForBlock_ Retrieve the replica information for a block. It takes several minutes after a block is injected for the statistics to be calculated, so this will block until that information is available. """ attempts = 0 while attempts < 15: result = self.phedex.getReplicaInfoForFiles(block=blockName) if "phedex" in result: if "block" in result["phedex"]: if len(result["phedex"]["block"]) != 0: return result["phedex"]["block"][0] attempts += 1 time.sleep(20) logging.info("Could not retrieve replica info for block: %s" % blockName) return None @attr("integration") def testPoller(self): """ _testPoller_ Stuff the database and have the poller upload files to PhEDEx. Retrieve replica information for the uploaded blocks and verify that all files have been injected. """ return self.stuffDatabase() poller = PhEDExInjectorPoller(self.createConfig()) poller.setup(parameters=None) poller.algorithm(parameters=None) replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName) goldenLFNs = [] for file in self.testFilesA: goldenLFNs.append(file["lfn"]) for replicaFile in replicaInfo["file"]: assert replicaFile["name"] in goldenLFNs, \ "Error: Extra file in replica block: %s" % replicaFile["name"] goldenLFNs.remove(replicaFile["name"]) assert len(goldenLFNs) == 0, \ "Error: Files missing from PhEDEx replica: %s" % goldenLFNs replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName) goldenLFNs = [] for file in self.testFilesB: goldenLFNs.append(file["lfn"]) for replicaFile in replicaInfo["file"]: assert replicaFile["name"] in goldenLFNs, \ "Error: Extra file in replica block: %s" % replicaFile["name"] goldenLFNs.remove(replicaFile["name"]) assert len(goldenLFNs) == 0, \ "Error: Files missing from PhEDEx replica: %s" % goldenLFNs myThread = threading.currentThread() daofactory = DAOFactory(package="WMComponent.DBSUpload.Database", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = daofactory(classname="SetBlockStatus") setBlock.execute(self.blockAName, locations=None, open_status="InGlobalDBS") poller.algorithm(parameters=None) replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName) assert replicaInfo["is_open"] == "n", \ "Error: block should be closed." replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName) assert replicaInfo["is_open"] == "y", \ "Error: block should be open." return def test_CustodialSiteA(self): """ _CustodialSiteA_ Check the custodialSite stuff by DAO, since I don't have a cert First make sure we properly handle having no custodialSite """ self.stuffDatabase() myThread = threading.currentThread() daofactory = DAOFactory(package="WMComponent.PhEDExInjector.Database", logger=myThread.logger, dbinterface=myThread.dbi) getUninjected = daofactory(classname="GetUninjectedFiles") uninjectedFiles = getUninjected.execute() self.assertEqual(uninjectedFiles.keys(), ['srm-cms.cern.ch']) return
class FeederManagerTest(unittest.TestCase): """ TestCase for TestFeederManager module """ _maxMessage = 10 def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all needed tables. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.generateWorkDir() self.testInit.setSchema(customModules = \ ['WMCore.Agent.Database', 'WMComponent.FeederManager.Database', 'WMCore.ThreadPool', 'WMCore.WMBS'], useDefault = False) return def tearDown(self): """ _tearDown_ Database deletion """ self.testInit.clearDatabase() return def getConfig(self): """ _createConfig_ Create a config for the JobAccountant. This config needs to include information for connecting to the database as the component will create it's own database connections. These parameters are still pulled from the environment. """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.component_("FeederManager") config.FeederManager.logLevel = "INFO" config.FeederManager.componentName = "FeederManager" config.FeederManager.componentDir = \ os.path.join(os.getenv("TESTDIR"), "FeederManager") config.FeederManager.addDatasetWatchHandler = \ 'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch' # The maximum number of threads to process each message type config.FeederManager.maxThreads = 10 # The poll interval at which to look for new fileset/feeder association config.FeederManager.pollInterval = 60 return config def testA(self): """ _testA_ Handle AddDatasetWatch events """ myThread = threading.currentThread() config = self.getConfig() testFeederManager = FeederManager(config) testFeederManager.prepareToStart() for i in xrange(0, FeederManagerTest._maxMessage): for j in xrange(0, 3): feederManagerdict = {'payload':{'FeederType':'NO Feeder', 'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE', 'StartRun' : 'NO START RUN' }} testFeederManager.handleMessage( type = 'AddDatasetWatch', payload = feederManagerdict ) time.sleep(30) myThread.workerThreadManager.terminateWorkers() while threading.activeCount() > 1: print('Currently: '+str(threading.activeCount())+\ ' Threads. Wait until all our threads have finished') time.sleep(1)
class WMAgentTest(unittest.TestCase): """ _WMAgentTest_ Global unittest for all WMAgent components """ # This is an integration test __integration__ = "Any old bollocks" sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN'] components = ['JobCreator', 'JobSubmitter', 'JobTracker', 'JobAccountant', 'JobArchiver', 'TaskArchiver', 'RetryManager', 'ErrorHandler'] def setUp(self): """ _setUp_ Set up vital components """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService', 'WMCore.ResourceControl', 'WMCore.ThreadPool', 'WMCore.Agent.Database'], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daoFactory(classname = "Locations.New") pendingSlots = self.daoFactory(classname = "Locations.SetPendingSlots") for site in self.sites: locationAction.execute(siteName = site, pnn = 'se.%s' % (site), ceName = site) pendingSlots.execute(siteName = site, pendingSlots = 1000) #Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, pnn = 'se.%s' % (site), ceName = site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) self.testDir = self.testInit.generateWorkDir() # Set heartbeat for component in self.components: heartbeatAPI = HeartbeatAPI(component) heartbeatAPI.registerComponent() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ Tear down everything and go home. """ self.testInit.clearDatabase() self.testInit.delWorkDir() EmulatorSetup.deleteConfig(self.configFile) return def createTestWorkload(self, workloadName = 'Test', emulator = True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("TestWorkload") rereco = workload.getTask("ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload def getConfig(self): """ _getConfig_ This is the global test configuration object """ config = self.testInit.getConfiguration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobCreator config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' config.JobCreator.logLevel = 'DEBUG' config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 2 config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components') # JobSubmitter config.component_("JobSubmitter") config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter' config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'CondorGlobusPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'DEBUG' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 # JobAccountant config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components') config.JobAccountant.logLevel = 'INFO' # JobArchiver config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components') config.JobArchiver.numberOfJobsToCluster = 1000 # Task Archiver config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testInit.generateWorkDir() config.TaskArchiver.WorkQueueParams = {} config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'mnorman:[email protected]:5984') config.JobStateMachine.couchDBName = "mnorman_test" # Needed, because this is a test os.makedirs(config.JobSubmitter.submitDir) return config def createFileCollection(self, name, nSubs, nFiles, workflowURL = 'test', site = None): """ _createFileCollection_ Create a collection of files for splitting into jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec = workflowURL, owner = "mnorman", name = name, task="/TestWorkload/ReReco") testWorkflow.create() for sub in range(nSubs): nameStr = '%s-%i' % (name, sub) testFileset = Fileset(name = nameStr) testFileset.create() for f in range(nFiles): # pick a random site if not site: tmpSite = 'se.%s' % (random.choice(self.sites)) else: tmpSite = 'se.%s' % (site) testFile = File(lfn = "/lfn/%s/%i" % (nameStr, f), size = 1024, events = 10) testFile.setLocation(tmpSite) testFile.create() testFileset.addFile(testFile) testFileset.commit() testFileset.markOpen(isOpen = 0) testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased") testSubscription.create() return def createReports(self, jobs, retryCount = 0): """ _createReports_ Create some dummy job reports for each job """ report = Report() report.addStep('testStep', 0) for job in jobs: #reportPath = os.path.join(job['cache_dir'], 'Report.%i.pkl' % (retryCount)) reportPath = job['fwjr_path'] if os.path.exists(reportPath): os.remove(reportPath) report.save(reportPath) return def testA_StraightThrough(self): """ _StraightThrough_ Just run everything straight through without any variations """ # Do pre-submit job check nRunning = getCondorRunningJobs() self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) myThread = threading.currentThread() workload = self.createTestWorkload() config = self.getConfig() name = 'WMAgent_Test1' site = self.sites[0] nSubs = 5 nFiles = 10 workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') # Create a collection of files self.createFileCollection(name = name, nSubs = nSubs, nFiles = nFiles, workflowURL = workloadPath, site = site) ############################################################ # Test the JobCreator config.Agent.componentName = 'JobCreator' testJobCreator = JobCreatorPoller(config = config) testJobCreator.algorithm() time.sleep(5) # Did all jobs get created? getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), nSubs*nFiles) # Count database objects result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco') self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory)) self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0') # First job should be in here self.assertTrue('job_1' in os.listdir(groupDirectory)) jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl') self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, 'r') job = pickle.load(f) f.close() self.assertEqual(job['workflow'], name) self.assertEqual(len(job['input_files']), 1) self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2') ############################################################### # Now test the JobSubmitter config.Agent.componentName = 'JobSubmitter' testJobSubmitter = JobSubmitterPoller(config = config) testJobSubmitter.algorithm() # Check that jobs are in the right state result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) # Check assigned locations getLocationAction = self.daoFactory(classname = "Jobs.GetLocation") for id in result: loc = getLocationAction.execute(jobid = id) self.assertEqual(loc, [[site]]) # Check to make sure we have running jobs nRunning = getCondorRunningJobs() self.assertEqual(nRunning, nFiles * nSubs) ################################################################# # Now the JobTracker config.Agent.componentName = 'JobTracker' testJobTracker = JobTrackerPoller(config = config) testJobTracker.setup() testJobTracker.algorithm() # Running the algo without removing the jobs should do nothing result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) condorRM() time.sleep(1) # All jobs gone? nRunning = getCondorRunningJobs() self.assertEqual(nRunning, 0) testJobTracker.algorithm() time.sleep(5) # Running the algo without removing the jobs should do nothing result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) ################################################################# # Now the JobAccountant # First you need to load all jobs self.getFWJRAction = self.daoFactory(classname = "Jobs.GetFWJRByState") completeJobs = self.getFWJRAction.execute(state = "complete") # Create reports for all jobs self.createReports(jobs = completeJobs, retryCount = 0) config.Agent.componentName = 'JobAccountant' testJobAccountant = JobAccountantPoller(config = config) testJobAccountant.setup() # It should do something with the jobs testJobAccountant.algorithm() # All the jobs should be done now result = getJobsAction.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Success', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) ####################################################################### # Now the JobArchiver config.Agent.componentName = 'JobArchiver' testJobArchiver = JobArchiverPoller(config = config) testJobArchiver.algorithm() # All the jobs should be cleaned up result = getJobsAction.execute(state = 'Success', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) logDir = os.path.join(self.testDir, 'logs') for job in completeJobs: self.assertFalse(os.path.exists(job['fwjr_path'])) jobFolder = 'JobCluster_%i' \ % (int(job['id']/config.JobArchiver.numberOfJobsToCluster)) jobPath = os.path.join(logDir, jobFolder, 'Job_%i.tar' %(job['id'])) self.assertTrue(os.path.isfile(jobPath)) self.assertTrue(os.path.getsize(jobPath) > 0) ########################################################################### # Now the TaskAchiver config.Agent.componentName = 'TaskArchiver' testTaskArchiver = TaskArchiverPoller(config = config) testTaskArchiver.algorithm() result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing") self.assertEqual(len(result), 0) for jdict in completeJobs: job = Job(id = jdict['id']) self.assertFalse(job.exists()) if os.path.isdir('testDir'): shutil.rmtree('testDir') shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir')) return
class ReportIntegrationTest(unittest.TestCase): """ _ReportIntegrationTest_ """ def setUp(self): """ _setUp_ Setup the database and WMBS for the test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daofactory(classname = "Locations.New") locationAction.execute(siteName = "site1", seName = "cmssrm.fnal.gov") inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10, locations = "cmssrm.fnal.gov") inputFile.create() inputFileset = Fileset(name = "InputFileset") inputFileset.create() inputFileset.addFile(inputFile) inputFileset.commit() unmergedFileset = Fileset(name = "UnmergedFileset") unmergedFileset.create() mergedFileset = Fileset(name = "MergedFileset") mergedFileset.create() procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve", name = "TestWF", task = "/TestWF/None") procWorkflow.create() procWorkflow.addOutput("outputRECORECO", unmergedFileset) mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve", name = "MergeWF", task = "/MergeWF/None") mergeWorkflow.create() mergeWorkflow.addOutput("Merged", mergedFileset) insertWorkflow = self.dbsfactory(classname = "InsertWorkflow") insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0) insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0) self.procSubscription = Subscription(fileset = inputFileset, workflow = procWorkflow, split_algo = "FileBased", type = "Processing") self.procSubscription.create() self.procSubscription.acquireFiles() self.mergeSubscription = Subscription(fileset = unmergedFileset, workflow = mergeWorkflow, split_algo = "WMBSMergeBySize", type = "Merge") self.mergeSubscription.create() self.procJobGroup = JobGroup(subscription = self.procSubscription) self.procJobGroup.create() self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription) self.mergeJobGroup.create() self.testJob = Job(name = "testJob", files = [inputFile]) self.testJob.create(group = self.procJobGroup) self.testJob["state"] = "complete" myThread = threading.currentThread() self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState") self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath") self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType") locationAction = self.daofactory(classname = "Locations.New") locationAction.execute(siteName = "cmssrm.fnal.gov") self.stateChangeAction.execute(jobs = [self.testJob]) self.tempDir = tempfile.mkdtemp() return def tearDown(self): """ _tearDown_ Clear out the database and the pickled report file. """ self.testInit.clearDatabase() try: os.remove(os.path.join(self.tempDir, "ProcReport.pkl")) os.remove(os.path.join(self.tempDir, "MergeReport.pkl")) except Exception as ex: pass try: os.rmdir(self.tempDir) except Exception as ex: pass return def createConfig(self, workerThreads): """ _createConfig_ Create a config for the JobAccountant with the given number of worker threads. This config needs to include information for connecting to the database as the component will create it's own database connections. These parameters are still pulled from the environment. """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = "report_integration_t" config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.workerThreads = workerThreads config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) return config def verifyJobSuccess(self, jobID): """ _verifyJobSuccess_ Verify that the metadata for a successful job is correct. This will check the outcome, retry count and state. """ testJob = Job(id = jobID) testJob.load() assert testJob["state"] == "success", \ "Error: test job in wrong state: %s" % testJob["state"] assert testJob["retry_count"] == 0, \ "Error: test job has wrong retry count: %s" % testJob["retry_count"] assert testJob["outcome"] == "success", \ "Error: test job has wrong outcome: %s" % testJob["outcome"] return def verifyFileMetaData(self, jobID, fwkJobReportFiles): """ _verifyFileMetaData_ Verify that all the files that were output by a job made it into WMBS correctly. Compare the contents of WMBS to the files in the frameworks job report. Note that fwkJobReportFiles is a list of DataStructs File objects. """ testJob = Job(id = jobID) testJob.loadData() inputLFNs = [] for inputFile in testJob["input_files"]: inputLFNs.append(inputFile["lfn"]) for fwkJobReportFile in fwkJobReportFiles: outputFile = File(lfn = fwkJobReportFile["lfn"]) outputFile.loadData(parentage = 1) assert outputFile["events"] == int(fwkJobReportFile["events"]), \ "Error: Output file has wrong events: %s, %s" % \ (outputFile["events"], fwkJobReportFile["events"]) assert outputFile["size"] == int(fwkJobReportFile["size"]), \ "Error: Output file has wrong size: %s, %s" % \ (outputFile["size"], fwkJobReportFile["size"]) for ckType in fwkJobReportFile["checksums"].keys(): assert ckType in outputFile["checksums"].keys(), \ "Error: Output file is missing checksums: %s" % ckType assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \ "Error: Checksums don't match." assert len(fwkJobReportFile["checksums"].keys()) == \ len(outputFile["checksums"].keys()), \ "Error: Wrong number of checksums." jobType = self.getJobTypeAction.execute(jobID = jobID) if jobType == "Merge": assert str(outputFile["merged"]) == "True", \ "Error: Merge jobs should output merged files." else: assert outputFile["merged"] == fwkJobReportFile["merged"], \ "Error: Output file merged output is wrong: %s, %s" % \ (outputFile["merged"], fwkJobReportFile["merged"]) assert len(outputFile["locations"]) == 1, \ "Error: outputfile should have one location: %s" % outputFile["locations"] assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \ "Error: wrong location for file." assert len(outputFile["parents"]) == len(inputLFNs), \ "Error: Output file has wrong number of parents." for outputParent in outputFile["parents"]: assert outputParent["lfn"] in inputLFNs, \ "Error: Unknown parent file: %s" % outputParent["lfn"] fwjrRuns = {} for run in fwkJobReportFile["runs"]: fwjrRuns[run.run] = run.lumis for run in outputFile["runs"]: assert run.run in fwjrRuns, \ "Error: Extra run in output: %s" % run.run for lumi in run: assert lumi in fwjrRuns[run.run], \ "Error: Extra lumi: %s" % lumi fwjrRuns[run.run].remove(lumi) if len(fwjrRuns[run.run]) == 0: del fwjrRuns[run.run] assert len(fwjrRuns.keys()) == 0, \ "Error: Missing runs, lumis: %s" % fwjrRuns testJobGroup = JobGroup(id = testJob["jobgroup"]) testJobGroup.loadData() jobGroupFileset = testJobGroup.output jobGroupFileset.loadData() assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \ "Error: output file not in jobgroup fileset." if testJob["mask"]["FirstEvent"] == None: assert outputFile["first_event"] == 0, \ "Error: first event not set correctly: 0, %s" % \ outputFile["first_event"] else: assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \ "Error: last event not set correctly: %s, %s" % \ (testJob["mask"]["FirstEvent"], outputFile["first_event"]) return def testReportHandling(self): """ _testReportHandling_ Verify that we're able to parse a CMSSW report, convert it to a Report() style report, pickle it and then have the accountant process it. """ self.procPath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(self.procPath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl") cmsRunStep = myReport.retrieveStep("cmsRun1") cmsRunStep.status = 0 myReport.setTaskName('/TestWF/None') myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath) pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000) pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") pFile.setDatasetPath("/bogus/dataset/path") #pFile.addRun(Run(1, *[45])) pFile.create() config = self.createConfig(workerThreads = 1) accountant = JobAccountantPoller(config) accountant.setup() accountant.algorithm() self.verifyJobSuccess(self.testJob["id"]) self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1")) inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root") inputFile.load() self.testMergeJob = Job(name = "testMergeJob", files = [inputFile]) self.testMergeJob.create(group = self.mergeJobGroup) self.testMergeJob["state"] = "complete" self.stateChangeAction.execute(jobs = [self.testMergeJob]) self.mergePath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml") myReport = Report("mergeReco") myReport.parse(self.mergePath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("mergeReco"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1", "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1", "dataTier": "RECO"} fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl") myReport.setTaskName('/MergeWF/None') cmsRunStep = myReport.retrieveStep("mergeReco") cmsRunStep.status = 0 myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath) accountant.algorithm() self.verifyJobSuccess(self.testMergeJob["id"]) self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco")) return
class ConfigTest(unittest.TestCase): """ Test class for the WMAgent configs """ def setUp(self): """ _setUp_ Create the whole database """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMComponent.DBSBuffer.Database", 'WMCore.ResourceControl', 'WMCore.BossAir'], useDefault = False) self.testDir = self.testInit.generateWorkDir() return def tearDown(self): """ _tearDown_ Tear things down and go home """ self.testInit.clearDatabase() return def initComponents(self, configPath): """ _initComponents_ Start up the various components using the config """ if os.path.isfile(configPath): # Read the config config = loadConfigurationFile(configPath) else: msg = "No config file at desired location" logging.error(msg) raise Exception(msg) masterConfig = self.testInit.getConfiguration() config.Agent.useHeartbeat = False config.CoreDatabase.socket = masterConfig.CoreDatabase.socket config.CoreDatabase.connectUrl = masterConfig.CoreDatabase.connectUrl # Have to do this because the agent hard codes its dirs oldWorkDir = config.General.workDir for compName in (config.listComponents_() + config.listWebapps_()): component = getattr(config, compName) for var in component.listSections_(): value = getattr(component, var) if type(value) == str: if re.search(oldWorkDir, value): # Then set it setattr(component, var, value.replace(oldWorkDir, self.testDir)) elif type(value) == list: # Go through it one component at a time for element in list: if type(element) == str and re.search(oldWorkDir, element): index = value.index(element) value.remove(element) value.insert(index, value.replace(oldWorkDir, self.testDir)) setattr(component, var, value) elif type(value) == dict: for key in value.keys(): if type(value[key]) == str and re.search(oldWorkDir, value[key]): value[key] = value[key].replace(oldWorkDir, self.testDir) setattr(component, var, value) compList = (config.listComponents_() + config.listWebapps_()) components = [] # Get all components components.append(JobCreator(config = config)) components.append(JobSubmitter(config = config)) components.append(JobTracker(config = config)) components.append(JobAccountant(config = config)) components.append(JobArchiver(config = config)) components.append(TaskArchiver(config = config)) components.append(ErrorHandler(config = config)) components.append(RetryManager(config = config)) components.append(DBSUpload(config = config)) # Now the optional ones if 'PhEDExInjector' in compList: components.append(PhEDExInjector(config = config)) # Init threads: for component in components: component.initInThread() # preInitialize for component in components: component.preInitialization() for component in components: component.prepareToStop() return def testA_WMAgentConfig(self): """ _WMAgentConfig_ Test the WMAgentConfig file in WMCORE/etc """ # Get the config configPath = os.path.join(WMCore.WMInit.getWMBASE(), 'etc', 'WMAgentConfig.py') self.initComponents(configPath = configPath) return @attr('integration') def testB_WMAgentPromptSkimConfig(self): """ _WMAgentConfig_ Test the WMAgentConfig file in WMCORE/etc """ # Get the config configPath = os.path.join(WMCore.WMInit.getWMBASE(), 'etc', 'WMAgentPromptSkimConfig.py') self.initComponents(configPath = configPath) return
class PhEDExInjectorSubscriberTest(unittest.TestCase): """ _PhEDExInjectorSubscriberTest_ Unit tests for the PhEDExInjectorSubscriber. Create some database inside DBSBuffer, run the subscriber algorithm using a PhEDEx emulator and verify that it works both in unsafe and safe mode. For unsafe mode there a WMBS database is also created """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() EmulatorHelper.resetEmulators() def createConfig(self, safeMode): """ _createConfig_ Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because we are using Emulators """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeDatasets = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 config.PhEDExInjector.safeOperationMode = safeMode return config def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database, both from the same workflow. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequest", "BogusTask", os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl")) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createBlock = uploadFactory(classname = "SetBlockStatus") self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 0) createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 0) bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask") return def stuffWMBS(self): """ _stuffWMBS_ Inject the workflow in WMBS and add the subscriptions """ testWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "") testWorkflow.create() testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "") testMergeWorkflow.create() testWMBSFileset = Fileset(name = "TopFileset") testWMBSFileset.create() testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset") testWMBSFilesetUnmerged.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFilesetUnmerged.addFile(testFileB) testWMBSFileset.commit() testWMBSFilesetUnmerged.commit() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged, workflow = testMergeWorkflow, type = "Merge") testSubscriptionMerge.create() return (testSubscription, testSubscriptionMerge) def testUnsafeModeSubscriptions(self): """ _testUnsafeModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on unsafe operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig(safeMode = False) subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed") return def testSafeModeSubscriptions(self): """ _testSafeModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on safe operation mode, make sure that the flow of subscriptions obeys the rule laid in the subscriber documentation. Check that the requests are correct. """ config = self.createConfig(safeMode = True) self.stuffDatabase() topSubscription, mergeSubscription = self.stuffWMBS() # Start the subscriber subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) # Run once, this means that all custodial and non-custodial subscriptions # will be made but none will be Move subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as partially subscribed") # Now finish the Processing subscription and run the algorithm again topSubscription.markFinished() subscriber.algorithm({}) self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed again to custodial site") moveCount = 0 for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) if subInfo["move"] == "y": moveCount += 1 else: self.fail("Dataset A was subscribed to a wrong site %s" % site) self.assertEqual(moveCount, 1, "Move subscription was not made") self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was susbcribed again") result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 2")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed") return
class ResultSetTest(unittest.TestCase): def setUp(self): """ Set up for initializing the ResultSet test class """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.testInit.setSchema(customModules = ["WMCore.WMBS"], # useDefault = False) self.mydialect = self.testInit.getConfiguration().CoreDatabase.dialect self.myThread = threading.currentThread() if self.mydialect.lower() == 'mysql': create_sql = "create table test (bind1 varchar(20), bind2 varchar(20)) ENGINE=InnoDB " elif self.mydialect.lower() == 'sqlite': create_sql = "create table test (bind1 varchar(20), bind2 varchar(20))" else: create_sql = "create table test (bind1 varchar(20), bind2 varchar(20))" #Create a table and insert several pieces self.myThread.dbi.processData(create_sql) return def tearDown(self): """ Delete the ResultSet test class """ sql = 'drop table test' self.myThread.dbi.processData(sqlstmt = sql, conn = self.myThread.dbi.connection()) return def testFullResultSet(self): """ Function to test all the functionality of the ResultSet """ print "testFullResultSet" #ResultSet is a very simple function. This function should #create a ResultSet, fill it with data, and then read it #out, all in one go. testSet = ResultSet() testDict = {'bind1': 'value1', 'bind2':'value2'} binds = [ {'bind1':'value3a', 'bind2': 'value2a'}, {'bind1':'value3b', 'bind2': 'value2b'}, {'bind1':'value3c', 'bind2': 'value2c'}] self.myThread.dbi.processData('insert into test (bind1, bind2) values (:bind1, :bind2)', testDict) self.myThread.dbi.processData('insert into test (bind1, bind2) values (:bind1, :bind2)', binds) testProxy = self.myThread.dbi.connection().execute('select * from test') #import pdb #pdb.set_trace() testSet.add(testProxy) self.assertEqual(testProxy.rowcount, 4) self.assertEqual(testSet.rowcount, 4) #Test to make sure fetchone and fetchall both work self.assertEqual(str(testSet.fetchone()[0]), 'value1') self.assertEqual(str(testSet.fetchall()[-1][1]), 'value2c') def testRowCount(self): testSet = ResultSet() insertProxy = self.myThread.dbi.connection().execute("insert into test (bind1, bind2) values ('a', 'b')") testSet.add(insertProxy) self.assertEqual(testSet.rowcount, 1) updateProxy = self.myThread.dbi.connection().execute("update test set bind1 = 'c'") testSet.add(updateProxy) self.assertEqual(testSet.rowcount, 2) updateProxy = self.myThread.dbi.connection().execute("update test set bind1 = 'c' where bind1 = 'a'") testSet.add(updateProxy) self.assertEqual(testSet.rowcount, 2) insertProxy = self.myThread.dbi.connection().execute("insert into test (bind1, bind2) values ('a', 'b')") testSet.add(insertProxy) self.assertEqual(testSet.rowcount, 3) selectProxy = self.myThread.dbi.connection().execute("select * from test") testSet.add(selectProxy) self.assertEqual(testSet.rowcount, 5) selectProxy = self.myThread.dbi.connection().execute("delete from test") testSet.add(selectProxy) self.assertEqual(testSet.rowcount, 7) def test1000Binds(self): testSet2 = ResultSet() #Now insert and select a huge number of values #This depends on the times variable binds = [ ] largeInsert = [ ] times = 1000 #For now I don't want to get too dependent on processData() with many binds, since right now #it doesn't work. That does make things awkward though. for i in range(times): binds = {'bind1': 'value1'+str(i), 'bind2':'value2'+str(i)} self.myThread.dbi.processData("insert into test (bind1, bind2) values (:bind1, :bind2)", binds) sql = "select bind1 from test" testProxy = self.myThread.dbi.connection().execute(sql) testSet2.add(testProxy) self.assertEqual(len(testSet2.fetchall()), times) if self.mydialect.lower() == 'sqlite': self.assertEqual(str(testSet2.fetchall()[1][0]), 'value11') self.assertEqual(testSet2.fetchone()[0], None) elif self.mydialect.lower() == 'mysql': self.assertEqual(str(testSet2.fetchall()[1][0]), 'value11') self.assertEqual(testSet2.fetchone()[0], 'value10') elif self.mydialect.lower() == 'oracle': self.assertEqual(str(testSet2.fetchall()[1][0]), 'value11') self.assertEqual(testSet2.fetchone()[0], 'value10') return
class TestRecoveryCode(unittest.TestCase): """ Test for recoveryCode; disaster recovery system """ #This is an integration test __integration__ = "So this guy walks into a bar..." def setUp(self): """ Mimic remains of destroyed job """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.nJobs = 10 os.mkdir('test') os.mkdir('test/Basic') os.mkdir('test/Basic/Crap') os.mkdir('test/Basic/Crap/JobCollection_1') for i in range(self.nJobs): os.mkdir('test/Basic/Crap/JobCollection_1/Job_%i' % (i)) self.logLevel = 'INFO' #There must be a submit directory for jobSubmitter if not os.path.isdir('submit'): os.mkdir('submit') #There must be a log directory for jobArchiver if not os.path.isdir('logs'): os.mkdir('logs') def tearDown(self): """ Check to make sure that everything is good. """ if os.path.isdir('test'): shutil.rmtree('test') for file in os.listdir('logs'): #You have to remove the staged out logs os.remove(os.path.join('logs', file)) def getConfig(self): """ _getConfig_ For now, build a config file from the ground up. Later, use this as a model for the JSM master config """ myThread = threading.currentThread() self.testInit = TestInit() self.testInit.setLogging() self.testInit.setDatabaseConnection() config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir( config = config ) #Now we go by component #First the JobCreator config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' config.JobCreator.logLevel = self.logLevel config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromSiteDB = True config.JobCreator.pollInterval = 10 config.JobCreator.jobCacheDir = os.path.join(os.getcwd(), 'test') config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 2 config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components/JobCreator') #JobMaker config.component_('JobMaker') config.JobMaker.logLevel = self.logLevel config.JobMaker.namespace = 'WMCore.WMSpec.Makers.JobMaker' config.JobMaker.maxThreads = 1 config.JobMaker.makeJobsHandler = 'WMCore.WMSpec.Makers.Handlers.MakeJobs' #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', 'mnorman:[email protected]:5984') config.JobStateMachine.couchDBName = "mnorman_test" #JobSubmitter config.component_("JobSubmitter") config.JobSubmitter.logLevel = self.logLevel config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'ShadowPoolPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(os.getcwd(), 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join(os.getcwd(), 'submit.sh') config.JobSubmitter.componentDir = os.path.join(os.getcwd(), 'Components/JobSubmitter') config.JobSubmitter.inputFile = os.path.join(os.getcwd(), 'FrameworkJobReport-4540.xml') config.JobSubmitter.workerThreads = 1 config.JobSubmitter.jobsPerWorker = 100 #JobTracker config.component_("JobTracker") config.JobTracker.logLevel = self.logLevel config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'TestTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 #ErrorHandler config.component_("ErrorHandler") config.ErrorHandler.logLevel = self.logLevel config.ErrorHandler.namespace = 'WMComponent.ErrorHandler.ErrorHandler' config.ErrorHandler.maxThreads = 30 config.ErrorHandler.maxRetries = 10 config.ErrorHandler.pollInterval = 10 #RetryManager config.component_("RetryManager") config.RetryManager.logLevel = self.logLevel config.RetryManager.namespace = 'WMComponent.RetryManager.RetryManager' config.RetryManager.maxRetries = 10 config.RetryManager.pollInterval = 10 config.RetryManager.coolOffTime = {'create': 10, 'submit': 10, 'job': 10} config.RetryManager.pluginPath = 'WMComponent.RetryManager.PlugIns' config.RetryManager.pluginName = '' config.RetryManager.WMCoreBase = WMCore.WMInit.getWMBASE() #JobAccountant config.component_("JobAccountant") config.JobAccountant.logLevel = self.logLevel #config.JobAccountant.logLevel = 'SQLDEBUG' config.JobAccountant.pollInterval = 10 config.JobAccountant.workerThreads = 1 config.JobAccountant.componentDir = os.path.join(os.getcwd(), 'Components/JobAccountant') #JobArchiver config.component_("JobArchiver") config.JobArchiver.pollInterval = 10 config.JobArchiver.logLevel = self.logLevel #config.JobArchiver.logLevel = 'SQLDEBUG' config.JobArchiver.logDir = os.path.join(os.getcwd(), 'logs') return config def submitJobs(self, nJobs): """ _submitJobs_ Submit some broken jdls to the local condor submitter """ submitFile = """ universe = globus globusscheduler = thisisadummyname.fnal.gov/jobmanager-suck should_transfer_executable = TRUE transfer_output_files = FrameworkJobReport.xml should_transfer_files = YES when_to_transfer_output = ON_EXIT log_xml = True notification = NEVER Output = condor.$(Cluster).$(Process).out Error = condor.$(Cluster).$(Process).err Log = condor.$(Cluster).$(Process).log Executable = /home/mnorman/WMCORE/test/python/WMCore_t/Operations_t/submit.sh initialdir = /home/mnorman/WMCORE/test/python/WMCore_t/Operations_t/test/Basic/Crap/JobCollection_1/Job_%i +WMAgent_JobName = \"65bf3894-d873-11de-9e40-0030482c2dd0-1\" +WMAgent_JobID = %i Queue 1 """ for i in range(10): f = open('submit/submit_%i.jdl' %(i), 'w') f.write(submitFile % (i, i)) f.close() command = ["condor_submit", 'submit/submit_%i.jdl' %(i)] pipe = Popen(command, stdout = PIPE, stderr = PIPE, shell = False) pipe.wait() @attr('integration') def testPurge(self): """ _testPurge_ Test the purge function, which should remove all job objects """ config = self.getConfig() self.submitJobs(self.nJobs) self.assertEqual(len(os.listdir('submit')), self.nJobs, 'Only found %i submit files' %(len(os.listdir('submit')))) self.assertEqual(len(os.listdir('logs')), 0, 'Please empty the logs directory') #Check that ten jobs were actually submitted jobCheckString = os.popen('condor_q %s' %os.getenv('USER')).readlines()[-1] self.assertEqual(jobCheckString, '%i jobs; %i idle, 0 running, 0 held\n' % (self.nJobs, self.nJobs)) purgeJobs = PurgeJobs(config) purgeJobs.run() self.assertEqual(os.listdir('test'), []) self.assertEqual(len(os.listdir('logs')), self.nJobs, \ 'Found %i tarballs instead of %i in logOut directory' \ %(len(os.listdir('logs')), self.nJobs) ) self.assertEqual(os.listdir('submit'), []) #Check that jobs were actually removed jobCheckString = os.popen('condor_q %s' %os.getenv('USER')).readlines()[-1] self.assertEqual(jobCheckString, '0 jobs; 0 idle, 0 running, 0 held\n' ) return
class FeederManagerTest(unittest.TestCase): """ TestCase for TestFeederManager module """ _maxMessage = 10 def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all needed tables. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.generateWorkDir() self.testInit.setSchema(customModules = \ ['WMCore.Agent.Database', 'WMComponent.FeederManager.Database', 'WMCore.ThreadPool', 'WMCore.WMBS'], useDefault = False) return def tearDown(self): """ _tearDown_ Database deletion """ self.testInit.clearDatabase() return def getConfig(self): """ _getConfig_ Get defaults FeederManager parameters """ return self.testInit.getConfiguration( os.path.join(WMCore.WMInit.getWMBASE(), \ 'src/python/WMComponent/FeederManager/DefaultConfig.py')) def testA(self): """ _testA_ Handle AddDatasetWatch events """ myThread = threading.currentThread() config = self.getConfig() testFeederManager = FeederManager(config) testFeederManager.prepareToStart() for i in xrange(0, FeederManagerTest._maxMessage): for j in xrange(0, 3): feederManagerdict = {'payload':{'FeederType':'NO Feeder', \ 'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE', \ 'StartRun' : 'NO START RUN' }} testFeederManager.handleMessage( type = 'AddDatasetWatch' \ , payload = feederManagerdict ) time.sleep(30) myThread.workerThreadManager.terminateWorkers() while threading.activeCount() > 1: print('Currently: '+str(threading.activeCount())+\ ' Threads. Wait until all our threads have finished') time.sleep(1)
class PhEDExInjectorSubscriberTest(unittest.TestCase): """ _PhEDExInjectorSubscriberTest_ Unit tests for the PhEDExInjectorSubscriber. Create some database inside DBSBuffer, run the subscriber algorithm using a PhEDEx emulator and verify that it works both in unsafe and safe mode. For unsafe mode there a WMBS database is also created """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() EmulatorHelper.resetEmulators() def createConfig(self, tier0Mode = False): """ _createConfig_ Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because we are using Emulators """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeDatasets = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 config.PhEDExInjector.tier0Mode = tier0Mode return config def stuffDatabase(self, tier0Mode = False): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createBlock = uploadFactory(classname = "SetBlockStatus") self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname = "NewSubscription") datasetA = DBSBufferDataset(path = self.testDatasetA) datasetB = DBSBufferDataset(path = self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) if tier0Mode: # Override the settings workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"], nonCustodialSites = ["T3_CO_Uniandes"], priority = "Normal", custodialSubType = "Replica", autoApproveSites = ["T0_CH_CERN"], dataTier = "RECO") workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"], nonCustodialSites = [], autoApproveSites = [], priority = "High", custodialSubType = "Replica", dataTier = "RAW") insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return def stuffWMBS(self): """ _stuffWMBS_ Inject a workflow in WMBS and add the subscriptions """ testWorkflow = Workflow(spec = "bogus.xml", owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequestB", task = "BogusTask", owner_vogroup = "", owner_vorole = "") testWorkflow.create() testMergeWorkflow = Workflow(spec = "bogus.xml", owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequestB", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "") testMergeWorkflow.create() testWMBSFileset = Fileset(name = "TopFileset") testWMBSFileset.create() testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset") testWMBSFilesetUnmerged.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFilesetUnmerged.addFile(testFileB) testWMBSFileset.commit() testWMBSFilesetUnmerged.commit() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged, workflow = testMergeWorkflow, type = "Merge") testSubscriptionMerge.create() return (testSubscription, testSubscriptionMerge) def testNormalModeSubscriptions(self): """ _testNormalModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on normal operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig() subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed") result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[0].fetchall() self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed") # Reset and run again and make sure that no duplicate subscriptions are created myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0") subscriber.algorithm({}) self.assertEqual(len(subscriptions[self.testDatasetA]), 3) self.assertEqual(len(subscriptions[self.testDatasetB]), 2) return def testTier0ModeSubscriptions(self): """ _testTier0ModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on tier0 operation mode, custodial moves are made on block level. """ self.stuffDatabase(tier0Mode = True) self.stuffWMBS() config = self.createConfig(tier0Mode = True) subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T0_CH_CERN and T1_US_FNAL # Non-custodial at T2_US_Vanderbilt # Autoapproved at CERN # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] if subInfo['level'] == 'block': self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["node"], "T0_CH_CERN_MSS", "Wrong node for dataset A, block subscription.") self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") elif site == "T0_CH_CERN_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") elif site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") else: self.fail("Dataset A was subscribed to a wrong site %s" % site) self.assertEqual(len([x for x in subInfoA if x['level'] == 'block']), 1) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, custodial to T0_CH_CERN_MSS and T1_UK_RAL_MSS # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions) subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] if site == "T0_CH_CERN_MSS" or site == "T1_UK_RAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 5, "Custodial move datasets were marked as subscribed") # Delete the wmbs entries myThread.dbi.processData("DELETE FROM wmbs_workflow") subscriber.algorithm({}) self.assertEqual(len(subscriptions[self.testDatasetA]), 5) self.assertTrue(self.testDatasetB in subscriptions) self.assertEqual(len(subscriptions[self.testDatasetB]), 3) self.assertEqual(len([x for x in subscriptions[self.testDatasetB] if x['level'] == 'block']), 1) return
class HarnessTest(unittest.TestCase): """ TestCase for TestComponent module """ tempDir = None def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() def tearDown(self): """ Delete database """ self.testInit.clearDatabase() def testB(self): raise nose.SkipTest config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir(config) config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent1") config.General.workDir = config.TestComponent.componentDir os.makedirs( config.TestComponent.componentDir ) # as this is a test we build the string from our global environment # parameters normally you put this straight into the DefaultConfig.py file: # testInit.getConfiguration returns from the environment variable by default testComponent = TestComponent(config) testComponent.prepareToStart() testComponent.handleMessage('LogState','') testComponent.handleMessage('TestMessage1','TestMessag1Payload') testComponent.handleMessage('TestMessage2','TestMessag2Payload') testComponent.handleMessage('TestMessage3','TestMessag3Payload') testComponent.handleMessage('TestMessage4','TestMessag4Payload') testComponent.handleMessage('Logging.DEBUG','') testComponent.handleMessage('Logging.WARNING','') testComponent.handleMessage('Logging.CRITICAL','') testComponent.handleMessage('Logging.ERROR','') testComponent.handleMessage('Logging.INFO','') testComponent.handleMessage('Logging.SQLDEBUG','') testComponent.handleMessage('TestComponent:Logging.DEBUG','') testComponent.handleMessage('TestComponent:Logging.WARNING','') testComponent.handleMessage('TestComponent:Logging.CRITICAL','') testComponent.handleMessage('TestComponent:Logging.ERROR','') testComponent.handleMessage('TestComponent:Logging.INFO','') testComponent.handleMessage('TestComponent:Logging.SQLDEBUG','') # test a non existing message (to generate an error) errorMsg = '' try: testComponent.handleMessage('NonExistingMessageType','') except Exception,ex: errorMsg = str(ex) self.assertTrue(errorMsg.startswith('Message NonExistingMessageType with payload'))
class PhEDExInjectorSubscriberTest(unittest.TestCase): """ _PhEDExInjectorSubscriberTest_ Unit tests for the PhEDExInjectorSubscriber. Create some database inside DBSBuffer, run the subscriber algorithm using a PhEDEx emulator and verify that it works both in unsafe and safe mode. For unsafe mode there a WMBS database is also created """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() EmulatorHelper.resetEmulators() def createConfig(self, tier0Mode = False): """ _createConfig_ Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because we are using Emulators """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeDatasets = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 config.PhEDExInjector.tier0Mode = tier0Mode return config def stuffDatabase(self, tier0Mode = False): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createBlock = uploadFactory(classname = "SetBlockStatus") self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 'Closed') bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname = "NewSubscription") datasetA = DBSBufferDataset(path = self.testDatasetA) datasetB = DBSBufferDataset(path = self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) if tier0Mode: # Override the settings workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_US_FNAL"], nonCustodialSites = ["T3_CO_Uniandes"], priority = "Normal", custodialSubType = "Replica", autoApproveSites = ["T0_CH_CERN"], dataTier = "RECO") workload.setSubscriptionInformation(custodialSites = ["T0_CH_CERN", "T1_UK_RAL"], nonCustodialSites = [], autoApproveSites = [], priority = "High", custodialSubType = "Replica", dataTier = "RAW") insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return def stuffWMBS(self): """ _stuffWMBS_ Inject a workflow in WMBS and add the subscriptions """ testWorkflow = Workflow(spec = "bogus.xml", owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequestB", task = "BogusTask", owner_vogroup = "", owner_vorole = "") testWorkflow.create() testMergeWorkflow = Workflow(spec = "bogus.xml", owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequestB", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "") testMergeWorkflow.create() testWMBSFileset = Fileset(name = "TopFileset") testWMBSFileset.create() testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset") testWMBSFilesetUnmerged.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFilesetUnmerged.addFile(testFileB) testWMBSFileset.commit() testWMBSFilesetUnmerged.commit() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged, workflow = testMergeWorkflow, type = "Merge") testSubscriptionMerge.create() return (testSubscription, testSubscriptionMerge) def testNormalModeSubscriptions(self): """ _testNormalModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on normal operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig() subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed") result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[0].fetchall() self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed") # Reset and run again and make sure that no duplicate subscriptions are created myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0") subscriber.algorithm({}) self.assertEqual(len(subscriptions[self.testDatasetA]), 3) self.assertEqual(len(subscriptions[self.testDatasetB]), 2) return def testTier0ModeSubscriptions(self): """ _testTier0ModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on tier0 operation mode, custodial moves are made on block level. """ self.stuffDatabase(tier0Mode = True) self.stuffWMBS() config = self.createConfig(tier0Mode = True) subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T0_CH_CERN and T1_US_FNAL # Non-custodial at T2_US_Vanderbilt # Autoapproved at CERN # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] if subInfo['level'] == 'block': self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["node"], "T0_CH_CERN_MSS", "Wrong node for dataset A, block subscription.") self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") elif site == "T0_CH_CERN_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") elif site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") else: self.fail("Dataset A was subscribed to a wrong site %s" % site) self.assertEqual(len([x for x in subInfoA if x['level'] == 'block']), 1) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, custodial to T0_CH_CERN_MSS and T1_UK_RAL_MSS # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions) subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] if site == "T0_CH_CERN_MSS" or site == "T1_UK_RAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["level"], "dataset", "Wrong level for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 5, "Custodial move datasets were marked as subscribed") # Delete the wmbs entries myThread.dbi.processData("DELETE FROM wmbs_workflow") subscriber.algorithm({}) self.assertEqual(len(subscriptions[self.testDatasetA]), 5) self.assertTrue(self.testDatasetB in subscriptions) self.assertEqual(len(subscriptions[self.testDatasetB]), 3) self.assertEqual(len([x for x in subscriptions[self.testDatasetB] if x['level'] == 'block']), 1) return
class WMAgentTest(unittest.TestCase): """ _WMAgentTest_ Global unittest for all WMAgent components """ # This is an integration test __integration__ = "Any old bollocks" sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN'] components = [ 'JobCreator', 'JobSubmitter', 'JobTracker', 'JobAccountant', 'JobArchiver', 'TaskArchiver', 'RetryManager', 'ErrorHandler' ] def setUp(self): """ _setUp_ Set up vital components """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=[ "WMCore.WMBS", 'WMCore.MsgService', 'WMCore.ResourceControl', 'WMCore.ThreadPool', 'WMCore.Agent.Database' ], useDefault=False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.daoFactory(classname="Locations.New") pendingSlots = self.daoFactory(classname="Locations.SetPendingSlots") for site in self.sites: locationAction.execute(siteName=site, seName='se.%s' % (site), ceName=site) pendingSlots.execute(siteName=site, pendingSlots=1000) #Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName=site, seName='se.%s' % (site), ceName=site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) self.testDir = self.testInit.generateWorkDir() # Set heartbeat for component in self.components: heartbeatAPI = HeartbeatAPI(component) heartbeatAPI.registerComponent() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ Tear down everything and go home. """ self.testInit.clearDatabase() self.testInit.delWorkDir() EmulatorSetup.deleteConfig(self.configFile) return def createTestWorkload(self, workloadName='Test', emulator=True): """ _createTestWorkload_ Creates a test workload for us to run on, hold the basic necessities. """ workload = testWorkload("TestWorkload") rereco = workload.getTask("ReReco") taskMaker = TaskMaker(workload, os.path.join(self.testDir, 'workloadTest')) taskMaker.skipSubscription = True taskMaker.processWorkload() workload.save(workloadName) return workload def getConfig(self): """ _getConfig_ This is the global test configuration object """ config = self.testInit.getConfiguration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = 'test' # First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) # Now the CoreDatabase information # This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobCreator config.component_("JobCreator") config.JobCreator.namespace = 'WMComponent.JobCreator.JobCreator' config.JobCreator.logLevel = 'DEBUG' config.JobCreator.maxThreads = 1 config.JobCreator.UpdateFromResourceControl = True config.JobCreator.pollInterval = 10 config.JobCreator.jobCacheDir = self.testDir config.JobCreator.defaultJobType = 'processing' #Type of jobs that we run, used for resource control config.JobCreator.workerThreads = 2 config.JobCreator.componentDir = os.path.join(os.getcwd(), 'Components') # JobSubmitter config.component_("JobSubmitter") config.JobSubmitter.namespace = 'WMComponent.JobSubmitter.JobSubmitter' config.JobSubmitter.logLevel = 'INFO' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.pluginName = 'CondorGlobusPlugin' config.JobSubmitter.pluginDir = 'JobSubmitter.Plugins' config.JobSubmitter.submitDir = os.path.join(self.testDir, 'submit') config.JobSubmitter.submitNode = os.getenv("HOSTNAME", 'badtest.fnal.gov') config.JobSubmitter.submitScript = os.path.join( getWMBASE(), 'test/python/WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( os.getcwd(), 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 # JobTracker config.component_("JobTracker") config.JobTracker.logLevel = 'DEBUG' config.JobTracker.pollInterval = 10 config.JobTracker.trackerName = 'CondorTracker' config.JobTracker.pluginDir = 'WMComponent.JobTracker.Plugins' config.JobTracker.componentDir = os.path.join(os.getcwd(), 'Components') config.JobTracker.runTimeLimit = 7776000 #Jobs expire after 90 days config.JobTracker.idleTimeLimit = 7776000 config.JobTracker.heldTimeLimit = 7776000 config.JobTracker.unknTimeLimit = 7776000 # JobAccountant config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.componentDir = os.path.join( os.getcwd(), 'Components') config.JobAccountant.logLevel = 'INFO' # JobArchiver config.component_("JobArchiver") config.JobArchiver.pollInterval = 60 config.JobArchiver.logLevel = 'INFO' config.JobArchiver.logDir = os.path.join(self.testDir, 'logs') config.JobArchiver.componentDir = os.path.join(os.getcwd(), 'Components') config.JobArchiver.numberOfJobsToCluster = 1000 # Task Archiver config.component_("TaskArchiver") config.TaskArchiver.componentDir = self.testInit.generateWorkDir() config.TaskArchiver.WorkQueueParams = {} config.TaskArchiver.pollInterval = 60 config.TaskArchiver.logLevel = 'INFO' config.TaskArchiver.timeOut = 0 # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv( 'COUCHURL', 'mnorman:[email protected]:5984') config.JobStateMachine.couchDBName = "mnorman_test" # Needed, because this is a test os.makedirs(config.JobSubmitter.submitDir) return config def createFileCollection(self, name, nSubs, nFiles, workflowURL='test', site=None): """ _createFileCollection_ Create a collection of files for splitting into jobs """ myThread = threading.currentThread() testWorkflow = Workflow(spec=workflowURL, owner="mnorman", name=name, task="/TestWorkload/ReReco") testWorkflow.create() for sub in range(nSubs): nameStr = '%s-%i' % (name, sub) testFileset = Fileset(name=nameStr) testFileset.create() for f in range(nFiles): # pick a random site if not site: tmpSite = 'se.%s' % (random.choice(self.sites)) else: tmpSite = 'se.%s' % (site) testFile = File(lfn="/lfn/%s/%i" % (nameStr, f), size=1024, events=10) testFile.setLocation(tmpSite) testFile.create() testFileset.addFile(testFile) testFileset.commit() testFileset.markOpen(isOpen=0) testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, type="Processing", split_algo="FileBased") testSubscription.create() return def createReports(self, jobs, retryCount=0): """ _createReports_ Create some dummy job reports for each job """ report = Report() report.addStep('testStep', 0) for job in jobs: #reportPath = os.path.join(job['cache_dir'], 'Report.%i.pkl' % (retryCount)) reportPath = job['fwjr_path'] if os.path.exists(reportPath): os.remove(reportPath) report.save(reportPath) return def testA_StraightThrough(self): """ _StraightThrough_ Just run everything straight through without any variations """ # Do pre-submit job check nRunning = getCondorRunningJobs() self.assertEqual( nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) myThread = threading.currentThread() workload = self.createTestWorkload() config = self.getConfig() name = 'WMAgent_Test1' site = self.sites[0] nSubs = 5 nFiles = 10 workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') # Create a collection of files self.createFileCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath, site=site) ############################################################ # Test the JobCreator config.Agent.componentName = 'JobCreator' testJobCreator = JobCreatorPoller(config=config) testJobCreator.algorithm() time.sleep(5) # Did all jobs get created? getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) # Count database objects result = myThread.dbi.processData( 'SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco') self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory)) self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0') # First job should be in here self.assertTrue('job_1' in os.listdir(groupDirectory)) jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl') self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, 'r') job = cPickle.load(f) f.close() self.assertEqual(job['workflow'], name) self.assertEqual(len(job['input_files']), 1) self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2') ############################################################### # Now test the JobSubmitter config.Agent.componentName = 'JobSubmitter' testJobSubmitter = JobSubmitterPoller(config=config) testJobSubmitter.algorithm() # Check that jobs are in the right state result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) # Check assigned locations getLocationAction = self.daoFactory(classname="Jobs.GetLocation") for id in result: loc = getLocationAction.execute(jobid=id) self.assertEqual(loc, [[site]]) # Check to make sure we have running jobs nRunning = getCondorRunningJobs() self.assertEqual(nRunning, nFiles * nSubs) ################################################################# # Now the JobTracker config.Agent.componentName = 'JobTracker' testJobTracker = JobTrackerPoller(config=config) testJobTracker.setup() testJobTracker.algorithm() # Running the algo without removing the jobs should do nothing result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) condorRM() time.sleep(1) # All jobs gone? nRunning = getCondorRunningJobs() self.assertEqual(nRunning, 0) testJobTracker.algorithm() time.sleep(5) # Running the algo without removing the jobs should do nothing result = getJobsAction.execute(state='Executing', jobType="Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Complete', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) ################################################################# # Now the JobAccountant # First you need to load all jobs self.getFWJRAction = self.daoFactory(classname="Jobs.GetFWJRByState") completeJobs = self.getFWJRAction.execute(state="complete") # Create reports for all jobs self.createReports(jobs=completeJobs, retryCount=0) config.Agent.componentName = 'JobAccountant' testJobAccountant = JobAccountantPoller(config=config) testJobAccountant.setup() # It should do something with the jobs testJobAccountant.algorithm() # All the jobs should be done now result = getJobsAction.execute(state='Complete', jobType="Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Success', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) ####################################################################### # Now the JobArchiver config.Agent.componentName = 'JobArchiver' testJobArchiver = JobArchiverPoller(config=config) testJobArchiver.algorithm() # All the jobs should be cleaned up result = getJobsAction.execute(state='Success', jobType="Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state='Cleanout', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) logDir = os.path.join(self.testDir, 'logs') for job in completeJobs: self.assertFalse(os.path.exists(job['fwjr_path'])) jobFolder = 'JobCluster_%i' \ % (int(job['id']/config.JobArchiver.numberOfJobsToCluster)) jobPath = os.path.join(logDir, jobFolder, 'Job_%i.tar' % (job['id'])) self.assertTrue(os.path.isfile(jobPath)) self.assertTrue(os.path.getsize(jobPath) > 0) ########################################################################### # Now the TaskAchiver config.Agent.componentName = 'TaskArchiver' testTaskArchiver = TaskArchiverPoller(config=config) testTaskArchiver.algorithm() result = getJobsAction.execute(state='Cleanout', jobType="Processing") self.assertEqual(len(result), 0) for jdict in completeJobs: job = Job(id=jdict['id']) self.assertFalse(job.exists()) if os.path.isdir('testDir'): shutil.rmtree('testDir') shutil.copytree('%s' % self.testDir, os.path.join(os.getcwd(), 'testDir')) return
class PhEDExInjectorSubscriberTest(unittest.TestCase): """ _PhEDExInjectorSubscriberTest_ Unit tests for the PhEDExInjectorSubscriber. Create some database inside DBSBuffer, run the subscriber algorithm using a PhEDEx emulator and verify that it works both in unsafe and safe mode. For unsafe mode there a WMBS database is also created """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" EmulatorHelper.setEmulators(phedex = True, dbs = True, siteDB = True) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() EmulatorHelper.resetEmulators() def createConfig(self, safeMode): """ _createConfig_ Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because we are using Emulators """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeDatasets = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 config.PhEDExInjector.safeOperationMode = safeMode return config def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database, both from the same workflow. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequest", "BogusTask", os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl")) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) createBlock = uploadFactory(classname = "SetBlockStatus") self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() createBlock.execute(block = self.blockAName, locations = ["srm-cms.cern.ch"], open_status = 0) createBlock.execute(block = self.blockBName, locations = ["srm-cms.cern.ch"], open_status = 0) bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname = "DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask") return def stuffWMBS(self): """ _stuffWMBS_ Inject the workflow in WMBS and add the subscriptions """ testWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "") testWorkflow.create() testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "") testMergeWorkflow.create() testWMBSFileset = Fileset(name = "TopFileset") testWMBSFileset.create() testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset") testWMBSFilesetUnmerged.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFilesetUnmerged.addFile(testFileB) testWMBSFileset.commit() testWMBSFilesetUnmerged.commit() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged, workflow = testMergeWorkflow, type = "Merge") testSubscriptionMerge.create() return (testSubscription, testSubscriptionMerge) def testUnsafeModeSubscriptions(self): """ _testUnsafeModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on unsafe operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig(safeMode = False) subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all non-custodial sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed") return def testSafeModeSubscriptions(self): """ _testSafeModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on safe operation mode, make sure that the flow of subscriptions obeys the rule laid in the subscriber documentation. Check that the requests are correct. """ config = self.createConfig(safeMode = True) self.stuffDatabase() topSubscription, mergeSubscription = self.stuffWMBS() # Start the subscriber subscriber = PhEDExInjectorSubscriber(config) subscriber.setup({}) # Run once, this means that all custodial and non-custodial subscriptions # will be made but none will be Move subscriber.algorithm({}) phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all non-custodial sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 1")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as partially subscribed") # Now finish the Processing subscription and run the algorithm again topSubscription.markFinished() subscriber.algorithm({}) self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 4, "Dataset A was not subscribed again to custodial site") moveCount = 0 for subInfo in subInfoA: site = subInfo["node"][0] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) if subInfo["move"] == "y": moveCount += 1 else: self.fail("Dataset A was subscribed to a wrong site %s" % site) self.assertEqual(moveCount, 1, "Move subscription was not made") self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was susbcribed again") result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset where subscribed = 2")[0].fetchall() self.assertEqual(result[0][0], 2, "Not all datasets were marked as subscribed") return
class HarnessTest(unittest.TestCase): """ TestCase for TestComponent module """ tempDir = None def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() def tearDown(self): """ Delete database """ self.testInit.clearDatabase() def testB(self): raise nose.SkipTest config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir(config) config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent1") config.General.workDir = config.TestComponent.componentDir os.makedirs( config.TestComponent.componentDir ) # as this is a test we build the string from our global environment # parameters normally you put this straight into the DefaultConfig.py file: # testInit.getConfiguration returns from the environment variable by default testComponent = TestComponent(config) testComponent.prepareToStart() testComponent.handleMessage('LogState','') testComponent.handleMessage('TestMessage1','TestMessag1Payload') testComponent.handleMessage('TestMessage2','TestMessag2Payload') testComponent.handleMessage('TestMessage3','TestMessag3Payload') testComponent.handleMessage('TestMessage4','TestMessag4Payload') testComponent.handleMessage('Logging.DEBUG','') testComponent.handleMessage('Logging.WARNING','') testComponent.handleMessage('Logging.CRITICAL','') testComponent.handleMessage('Logging.ERROR','') testComponent.handleMessage('Logging.INFO','') testComponent.handleMessage('Logging.SQLDEBUG','') testComponent.handleMessage('TestComponent:Logging.DEBUG','') testComponent.handleMessage('TestComponent:Logging.WARNING','') testComponent.handleMessage('TestComponent:Logging.CRITICAL','') testComponent.handleMessage('TestComponent:Logging.ERROR','') testComponent.handleMessage('TestComponent:Logging.INFO','') testComponent.handleMessage('TestComponent:Logging.SQLDEBUG','') # test a non existing message (to generate an error) errorMsg = '' try: testComponent.handleMessage('NonExistingMessageType','') except Exception as ex: errorMsg = str(ex) self.assertTrue(errorMsg.startswith('Message NonExistingMessageType with payload')) def testC(self): raise nose.SkipTest config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir(config) config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") # try starting a component as a daemon: config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent1") os.makedirs( config.TestComponent.componentDir ) testComponent = TestComponent(config) # we set the parent to true as we are testing testComponent.startDaemon(keepParent = True) print('trying to kill the component') time.sleep(2) daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml") details = Details(daemonFile) print('Is component alive: '+str(details.isAlive())) time.sleep(2) details.killWithPrejudice() print('Daemon killed') def testD(self): raise nose.SkipTest config = self.testInit.getConfiguration() config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") self.tempDir = self.testInit.generateWorkDir(config) # try starting a component as a daemon: config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent2") os.makedirs( config.TestComponent.componentDir ) testComponent = TestComponent(config) # we set the parent to true as we are testing testComponent.startDaemon(keepParent = True) time.sleep(2) daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml") details = Details(daemonFile) print('Is component alive: '+str(details.isAlive())) #create msgService to send stop message. myThread = threading.currentThread() factory = WMFactory("msgService", "WMCore.MsgService."+ \ myThread.dialect) myThread.transaction = Transaction(myThread.dbi) msgService = factory.loadObject("MsgService") msgService.registerAs("HarnessTest") myThread.transaction.commit() print('Publish a stop message to test if the component shutsdown gracefully') myThread.transaction.begin() msg = {'name' : 'Stop', 'payload' : ''} msgService.publish(msg) myThread.transaction.commit() msgService.finish() while details.isAlive(): print('Component has not received stop message') time.sleep(2) print('Daemon shutdown gracefully')
class HarvestTest(unittest.TestCase): """ _HarvestTest_ Test for EndOfRun job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE") myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE2", "SomeCE") myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE3", "SomeCE2") self.fileset1 = Fileset(name = "TestFileset1") for file in range(11): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Harvest", type = "Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = 'wmagent_jobdump' return config def finishJobs(self, jobGroups, subscription = None): """ _finishJobs_ """ if not subscription: subscription = self.subscription1 for f in subscription.acquiredFiles(): subscription.completeFiles(f) for jobGroup in jobGroups: self.changer.propagate(jobGroup.jobs, 'executing', 'created') self.changer.propagate(jobGroup.jobs, 'complete', 'executing') self.changer.propagate(jobGroup.jobs, 'success', 'complete') self.changer.propagate(jobGroup.jobs, 'cleanout', 'success') return def testHarvestEndOfRunTrigger(self): """ _testDQMHarvestEndOfRunTrigger_ Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset """ self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't") jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 0 , "We got 1 or more jobGroups with an open fileset and no periodic configuration") self.fileset1.markOpen(False) self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?") # We should also check if there are aqcuired files, if there are, there are jobs, # we don't want to fire another jobs while previous are running (output is integrating whatever input) # TODO : The above one we can do when all is done. Not priority jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1 , "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed") return def testPeriodicTrigger(self): """ _testPeriodicTrigger_ """ self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here") # Test timeout (5s for this first test) # there should be no acquired files, if there are, shouldn't be a job #self.subscription1.acquireFiles(self.subscription1.availableFiles().pop()) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 3) self.assertEqual(len(jobGroups), 1 , "Didn't created the first periodic job when there were acquired files") # For the whole thing to work, faking the first job finishing, and putting the files as complete self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for file in range(12,24): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.commit() # Testing that it doesn't create a job unless the delay is past jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 0 , "Created one or more job, when there were non-acquired file and the period is not passed by") time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, and there weren't and the period is passed by") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for file in range(26,36): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.commit() # Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 0 , "Created one or more job, there are new files, but the delay is not past") time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1 , "Didn't created one or more job, there are new files and the delay is past") # Last check is whether the job gets all the files or not numFilesJob = jobGroups[0].jobs[0].getFiles() numFilesFileset = self.fileset1.getFiles() self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding files for the first location for file in range(38,48): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.commit() # Then another location for file in range(50,56): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE3') self.fileset1.addFile(newFile) self.fileset1.commit() # We should have jobs in both locations time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups[0].getJobs()), 2 , "We didn't get 2 jobs for 2 locations") firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0] secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0] self.assertEqual(firstJobLocation, 'SomeSite', "First job location is not SomeSite") self.assertEqual(secondJobLocation, 'SomeSite2', "Second job location is not SomeSite2") self.finishJobs(jobGroups) for file in range(60,65): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(2,*[2])) newFile.setLocation('SomeSE3') self.fileset1.addFile(newFile) self.fileset1.commit() for file in range(70,75): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(3,*[3])) newFile.setLocation('SomeSE3') self.fileset1.addFile(newFile) self.fileset1.commit() time.sleep(2) jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = self.subscription1) jobGroups = jobFactory(periodic_harvest_interval = 2) # This is one of the most "complicated" tests so worth to comment, 4 jobs should be created # 1 - all previous files from SomeSE and run = 1 (a lot, like ~45) # 2 - Few files from SomeSE3, Run = 1 # 3 - Few files from SomeSE3, Run = 2 # 4 - Few files from SomeSE3, Run = 3 self.assertEqual(len(jobGroups[0].getJobs()), 4 , "We didn't get 4 jobs for adding 2 different runs to SomeSE3") return def testMultipleRunHarvesting(self): """ _testMultipleRunHarvesting_ Add some files with multiple runs in each, make sure the jobs are created by location and run. Verify each job mask afterwards. Note that in this test run are splitted between sites, in real life that MUST NOT happen we still don't support that. """ multipleFilesFileset = Fileset(name = "TestFileset") newFile = File("/some/file/test1", size = 1000, events = 100) newFile.addRun(Run(1,*[1,3,4,5,6,7])) newFile.addRun(Run(2,*[1,2,4,5,6,7])) newFile.setLocation('SomeSE') multipleFilesFileset.addFile(newFile) newFile = File("/some/file/test2", size = 1000, events = 100) newFile.addRun(Run(1,*[2,8])) newFile.addRun(Run(2,*[3,8])) newFile.setLocation('SomeSE3') multipleFilesFileset.addFile(newFile) multipleFilesFileset.create() harvestingWorkflow = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset = multipleFilesFileset, workflow = harvestingWorkflow, split_algo = "Harvest", type = "Harvesting") harvestSub.create() jobFactory = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSub) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask") self.finishJobs(jobGroups, harvestSub) newFile = File("/some/file/test3", size = 1000, events = 100) newFile.addRun(Run(1,*range(9,15))) newFile.setLocation('SomeSE3') multipleFilesFileset.addFile(newFile) multipleFilesFileset.commit() time.sleep(2) jobGroups = jobFactory(periodic_harvest_interval = 2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask") harvestingWorkflowSib = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflowSib", task="TestSib") harvestingWorkflowSib.create() harvestSubSib = Subscription(fileset = multipleFilesFileset, workflow = harvestingWorkflowSib, split_algo = "Harvest", type = "Harvesting") harvestSubSib.create() jobFactorySib = self.splitterFactory(package = "WMCore.WMBS", subscription = harvestSubSib) multipleFilesFileset.markOpen(False) jobGroups = jobFactorySib(periodic_harvest_sibling = True) self.assertEqual(len(jobGroups), 0, "A single job group was created") self.finishJobs(jobGroups, harvestSub) jobGroups = jobFactorySib(periodic_harvest_sibling = True) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") possibleLumiPairs = {1 : [[1,1],[3,7],[2,2],[8,8],[9,14]], 2 : [[1,2],[4,7],[3,3],[8,8]]} run = runs.keys()[0] for lumiPair in runs[run]: self.assertTrue(lumiPair in possibleLumiPairs[run], "Strange lumi pair in the job mask")
class HarnessTest(unittest.TestCase): """ TestCase for TestComponent module """ tempDir = None def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() def tearDown(self): """ Delete database """ self.testInit.clearDatabase() def testB(self): raise nose.SkipTest config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir(config) config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent1") config.General.workDir = config.TestComponent.componentDir os.makedirs(config.TestComponent.componentDir) # as this is a test we build the string from our global environment # parameters normally you put this straight into the DefaultConfig.py file: # testInit.getConfiguration returns from the environment variable by default testComponent = TestComponent(config) testComponent.prepareToStart() testComponent.handleMessage('LogState', '') testComponent.handleMessage('TestMessage1', 'TestMessag1Payload') testComponent.handleMessage('TestMessage2', 'TestMessag2Payload') testComponent.handleMessage('TestMessage3', 'TestMessag3Payload') testComponent.handleMessage('TestMessage4', 'TestMessag4Payload') testComponent.handleMessage('Logging.DEBUG', '') testComponent.handleMessage('Logging.WARNING', '') testComponent.handleMessage('Logging.CRITICAL', '') testComponent.handleMessage('Logging.ERROR', '') testComponent.handleMessage('Logging.INFO', '') testComponent.handleMessage('Logging.SQLDEBUG', '') testComponent.handleMessage('TestComponent:Logging.DEBUG', '') testComponent.handleMessage('TestComponent:Logging.WARNING', '') testComponent.handleMessage('TestComponent:Logging.CRITICAL', '') testComponent.handleMessage('TestComponent:Logging.ERROR', '') testComponent.handleMessage('TestComponent:Logging.INFO', '') testComponent.handleMessage('TestComponent:Logging.SQLDEBUG', '') # test a non existing message (to generate an error) errorMsg = '' try: testComponent.handleMessage('NonExistingMessageType', '') except Exception as ex: errorMsg = str(ex) self.assertTrue(errorMsg.startswith('Message NonExistingMessageType with payload')) def testC(self): raise nose.SkipTest config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir(config) config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") # try starting a component as a daemon: config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent1") os.makedirs(config.TestComponent.componentDir) testComponent = TestComponent(config) # we set the parent to true as we are testing testComponent.startDaemon(keepParent=True) print('trying to kill the component') time.sleep(2) daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml") details = Details(daemonFile) print('Is component alive: ' + str(details.isAlive())) time.sleep(2) details.killWithPrejudice() print('Daemon killed') def testD(self): raise nose.SkipTest config = self.testInit.getConfiguration() config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") self.tempDir = self.testInit.generateWorkDir(config) # try starting a component as a daemon: config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent2") os.makedirs(config.TestComponent.componentDir) testComponent = TestComponent(config) # we set the parent to true as we are testing testComponent.startDaemon(keepParent=True) time.sleep(2) daemonFile = os.path.join(config.TestComponent.componentDir, "Daemon.xml") details = Details(daemonFile) print('Is component alive: ' + str(details.isAlive())) # create msgService to send stop message. myThread = threading.currentThread() factory = WMFactory("msgService", "WMCore.MsgService." + \ myThread.dialect) myThread.transaction = Transaction(myThread.dbi) msgService = factory.loadObject("MsgService") msgService.registerAs("HarnessTest") myThread.transaction.commit() print('Publish a stop message to test if the component shutsdown gracefully') myThread.transaction.begin() msg = {'name': 'Stop', 'payload': ''} msgService.publish(msg) myThread.transaction.commit() msgService.finish() while details.isAlive(): print('Component has not received stop message') time.sleep(2) print('Daemon shutdown gracefully')
class RucioInjectorPollerTest(EmulatedUnitTestCase): """ Tests for the RucioInjectorPoller component """ def setUp(self): """ Install the DBSBuffer schema into the database and connect to Rucio. """ super(RucioInjectorPollerTest, self).setUp() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) myThread = threading.currentThread() daofactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daofactory(classname="DBSBufferFiles.AddLocation") self.locations = ["T2_CH_CERN", "T1_US_FNAL_Disk"] for rse in self.locations: locationAction.execute(siteName=rse) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/SampleA/PromptReco-v1/RECO" self.testDatasetB = "/SampleB/CRUZET11-v1/RAW" if PY3: self.assertItemsEqual = self.assertCountEqual return def tearDown(self): """ Delete the database. """ self.testInit.clearDatabase() def createConfig(self): """ Create a basic configuration for the component """ config = self.testInit.getConfiguration() config.component_("RucioInjector") config.RucioInjector.pollInterval = 300 config.RucioInjector.pollIntervalRules = 43200 config.RucioInjector.cacheExpiration = 2 * 24 * 60 * 60 # two days config.RucioInjector.createBlockRules = True config.RucioInjector.RSEPostfix = False # enable it to append _Test to the RSE names config.RucioInjector.metaDIDProject = "Production" config.RucioInjector.containerDiskRuleParams = {"weight": "ddm_quota", "copies": 2, "grouping": "DATASET"} config.RucioInjector.containerDiskRuleRSEExpr = "(tier=2|tier=1)&cms_type=real&rse_type=DISK" config.RucioInjector.rucioAccount = "wma_test" config.RucioInjector.rucioUrl = "http://cmsrucio-int.cern.ch" config.RucioInjector.rucioAuthUrl = "https://cmsrucio-auth-int.cern.ch" return config def stuffDatabase(self): """ Fill the dbsbuffer tables with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. We'll inject files with the location set as an SE name as well as a PhEDEx node name as well. """ myThread = threading.currentThread() # Create the DAOs factory and the relevant instances buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = buffer3Factory(classname="DBSBufferFiles.SetBlock") fileStatus = buffer3Factory(classname="DBSBufferFiles.SetStatus") associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile") insertWorkflow = buffer3Factory(classname="InsertWorkflow") datasetAction = buffer3Factory(classname="NewDataset") createAction = buffer3Factory(classname="CreateBlocks") # Create workflow in the database insertWorkflow.execute("BogusRequest", "BogusTask", 0, 0, 0, 0) # First file on first block checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["T2_CH_CERN"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() # Second file on first block testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["T2_CH_CERN"])) testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() # Third file on first block testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["T2_CH_CERN"])) testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) # First file on second block testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["T1_US_FNAL_Disk"])) testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() # Second file on second block testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["T1_US_FNAL_Disk"])) testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) # insert datasets in the dbsbuffer table datasetAction.execute(datasetPath=self.testDatasetA) datasetAction.execute(datasetPath=self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() # create and insert blocks into dbsbuffer table newBlockA = DBSBufferBlock(name=self.blockAName, location="T2_CH_CERN", datasetpath=None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBufferBlock(name=self.blockBName, location="T1_US_FNAL_Disk", datasetpath=None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks=[newBlockA, newBlockB]) # associate files to their correspondent block id setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) # set file status to LOCAL fileStatus.execute(testFileA["lfn"], "LOCAL") fileStatus.execute(testFileB["lfn"], "LOCAL") fileStatus.execute(testFileC["lfn"], "LOCAL") fileStatus.execute(testFileD["lfn"], "LOCAL") fileStatus.execute(testFileE["lfn"], "LOCAL") # associate files to a given workflow associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask") return def testBadConfig(self): """ Test wrong component configuration """ config = self.createConfig() config.RucioInjector.metaDIDProject = "Very invalid project name" with self.assertRaises(RucioInjectorException): RucioInjectorPoller(config) def testActivityMap(self): """ Initialize a RucioInjectorPoller object and test `_activityMap` method """ poller = RucioInjectorPoller(self.createConfig()) # test production agent and non-Tape endpoint activity = poller._activityMap("T1_US_FNAL_Disk") self.assertEquals(activity, "Production Output") activity = poller._activityMap("T1_US_FNAL_Test") self.assertEquals(activity, "Production Output") # test production agent and Tape endpoint (which is forbidden at the moment) with self.assertRaises(WMRucioException): poller._activityMap("T1_US_FNAL_Tape") # now pretend it to be a T0 agent/component poller.isT0agent = True # test T0 agent and non-Tape endpoint activity = poller._activityMap("T1_US_FNAL_Disk") self.assertEquals(activity, "T0 Export") activity = poller._activityMap("T1_US_FNAL_Test") self.assertEquals(activity, "T0 Export") # test T0 agent and Tape endpoint activity = poller._activityMap("T1_US_FNAL_Tape") self.assertEquals(activity, "T0 Tape") def testLoadingFiles(self): """ Initialize a RucioInjectorPoller object and load uninjected files """ self.stuffDatabase() poller = RucioInjectorPoller(self.createConfig()) poller.setup(parameters=None) uninjectedFiles = poller.getUninjected.execute() self.assertItemsEqual(list(uninjectedFiles), self.locations) self.assertEquals(list(uninjectedFiles["T2_CH_CERN"]), [self.testDatasetA]) self.assertEquals(list(uninjectedFiles["T1_US_FNAL_Disk"]), [self.testDatasetB])
class ProcessPoolTest(unittest.TestCase): def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase = True) self.testInit.setSchema(customModules = ["WMCore.Agent.Database"], useDefault = False) return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() return def testA_ProcessPool(self): """ _testProcessPool_ """ raise nose.SkipTest config = self.testInit.getConfiguration() config.Agent.useHeartbeat = False self.testInit.generateWorkDir(config) processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker", totalSlaves = 1, componentDir = config.General.workDir, config = config, namespace = "WMCore_t") processPool.enqueue(["One", "Two", "Three"]) result = processPool.dequeue(3) self.assertEqual(len(result), 3, "Error: Expected three items back.") self.assertTrue( "One" in result) self.assertTrue( "Two" in result) self.assertTrue( "Three" in result) return def testB_ProcessPoolStress(self): """ _testProcessPoolStress_ """ raise nose.SkipTest config = self.testInit.getConfiguration() config.Agent.useHeartbeat = False self.testInit.generateWorkDir(config) processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker", totalSlaves = 1, componentDir = config.General.workDir, namespace = "WMCore_t", config = config) result = None input = None for i in range(1000): input = [] while i > 0: input.append("COMMAND%s" % i) i -= 1 processPool.enqueue(input) result = processPool.dequeue(len(input)) self.assertEqual(len(result), len(input), "Error: Wrong number of results returned.") for k in result: self.assertTrue(k in input) return def testC_MultiPool(self): """ _testMultiPool_ Run a test with multiple workers """ raise nose.SkipTest config = self.testInit.getConfiguration() config.Agent.useHeartbeat = False self.testInit.generateWorkDir(config) processPool = ProcessPool("ProcessPool_t.ProcessPoolTestWorker", totalSlaves = 3, componentDir = config.General.workDir, namespace = "WMCore_t", config = config) for i in range(100): input = [] while i > 0: input.append("COMMAND%s" % i) i -= 1 processPool.enqueue(input) result = processPool.dequeue(len(input)) self.assertEqual(len(result), len(input), "Error: Wrong number of results returned.")
class ReportIntegrationTest(unittest.TestCase): """ _ReportIntegrationTest_ """ def setUp(self): """ _setUp_ Setup the database and WMBS for the test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault = False) myThread = threading.currentThread() self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.dbsfactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daofactory(classname = "Locations.New") locationAction.execute(siteName = "site1", pnn = "T1_US_FNAL_Disk") inputFile = File(lfn = "/path/to/some/lfn", size = 10, events = 10, locations = "T1_US_FNAL_Disk") inputFile.create() inputFileset = Fileset(name = "InputFileset") inputFileset.create() inputFileset.addFile(inputFile) inputFileset.commit() unmergedFileset = Fileset(name = "UnmergedFileset") unmergedFileset.create() mergedFileset = Fileset(name = "MergedFileset") mergedFileset.create() procWorkflow = Workflow(spec = "wf001.xml", owner = "Steve", name = "TestWF", task = "/TestWF/None") procWorkflow.create() procWorkflow.addOutput("outputRECORECO", unmergedFileset) mergeWorkflow = Workflow(spec = "wf002.xml", owner = "Steve", name = "MergeWF", task = "/MergeWF/None") mergeWorkflow.create() mergeWorkflow.addOutput("Merged", mergedFileset) insertWorkflow = self.dbsfactory(classname = "InsertWorkflow") insertWorkflow.execute("TestWF", "/TestWF/None", 0, 0, 0, 0) insertWorkflow.execute("MergeWF", "/MergeWF/None", 0, 0, 0, 0) self.procSubscription = Subscription(fileset = inputFileset, workflow = procWorkflow, split_algo = "FileBased", type = "Processing") self.procSubscription.create() self.procSubscription.acquireFiles() self.mergeSubscription = Subscription(fileset = unmergedFileset, workflow = mergeWorkflow, split_algo = "WMBSMergeBySize", type = "Merge") self.mergeSubscription.create() self.procJobGroup = JobGroup(subscription = self.procSubscription) self.procJobGroup.create() self.mergeJobGroup = JobGroup(subscription = self.mergeSubscription) self.mergeJobGroup.create() self.testJob = Job(name = "testJob", files = [inputFile]) self.testJob.create(group = self.procJobGroup) self.testJob["state"] = "complete" myThread = threading.currentThread() self.daofactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.stateChangeAction = self.daofactory(classname = "Jobs.ChangeState") self.setFWJRAction = self.daofactory(classname = "Jobs.SetFWJRPath") self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType") locationAction = self.daofactory(classname = "Locations.New") locationAction.execute(siteName = "cmssrm.fnal.gov") self.stateChangeAction.execute(jobs = [self.testJob]) self.tempDir = tempfile.mkdtemp() return def tearDown(self): """ _tearDown_ Clear out the database and the pickled report file. """ self.testInit.clearDatabase() try: os.remove(os.path.join(self.tempDir, "ProcReport.pkl")) os.remove(os.path.join(self.tempDir, "MergeReport.pkl")) except Exception as ex: pass try: os.rmdir(self.tempDir) except Exception as ex: pass return def createConfig(self, workerThreads): """ _createConfig_ Create a config for the JobAccountant with the given number of worker threads. This config needs to include information for connecting to the database as the component will create it's own database connections. These parameters are still pulled from the environment. """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = "report_integration_t" config.JobStateMachine.jobSummaryDBName = "report_integration_wmagent_summary_t" config.component_("JobAccountant") config.JobAccountant.pollInterval = 60 config.JobAccountant.workerThreads = workerThreads config.JobAccountant.componentDir = os.getcwd() config.JobAccountant.logLevel = 'SQLDEBUG' config.component_("TaskArchiver") config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName) return config def verifyJobSuccess(self, jobID): """ _verifyJobSuccess_ Verify that the metadata for a successful job is correct. This will check the outcome, retry count and state. """ testJob = Job(id = jobID) testJob.load() assert testJob["state"] == "success", \ "Error: test job in wrong state: %s" % testJob["state"] assert testJob["retry_count"] == 0, \ "Error: test job has wrong retry count: %s" % testJob["retry_count"] assert testJob["outcome"] == "success", \ "Error: test job has wrong outcome: %s" % testJob["outcome"] return def verifyFileMetaData(self, jobID, fwkJobReportFiles): """ _verifyFileMetaData_ Verify that all the files that were output by a job made it into WMBS correctly. Compare the contents of WMBS to the files in the frameworks job report. Note that fwkJobReportFiles is a list of DataStructs File objects. """ testJob = Job(id = jobID) testJob.loadData() inputLFNs = [] for inputFile in testJob["input_files"]: inputLFNs.append(inputFile["lfn"]) for fwkJobReportFile in fwkJobReportFiles: outputFile = File(lfn = fwkJobReportFile["lfn"]) outputFile.loadData(parentage = 1) assert outputFile["events"] == int(fwkJobReportFile["events"]), \ "Error: Output file has wrong events: %s, %s" % \ (outputFile["events"], fwkJobReportFile["events"]) assert outputFile["size"] == int(fwkJobReportFile["size"]), \ "Error: Output file has wrong size: %s, %s" % \ (outputFile["size"], fwkJobReportFile["size"]) for ckType in fwkJobReportFile["checksums"]: assert ckType in outputFile["checksums"], \ "Error: Output file is missing checksums: %s" % ckType assert outputFile["checksums"][ckType] == fwkJobReportFile["checksums"][ckType], \ "Error: Checksums don't match." assert len(fwkJobReportFile["checksums"]) == \ len(outputFile["checksums"]), \ "Error: Wrong number of checksums." jobType = self.getJobTypeAction.execute(jobID = jobID) if jobType == "Merge": assert str(outputFile["merged"]) == "True", \ "Error: Merge jobs should output merged files." else: assert outputFile["merged"] == fwkJobReportFile["merged"], \ "Error: Output file merged output is wrong: %s, %s" % \ (outputFile["merged"], fwkJobReportFile["merged"]) assert len(outputFile["locations"]) == 1, \ "Error: outputfile should have one location: %s" % outputFile["locations"] assert list(outputFile["locations"])[0] == list(fwkJobReportFile["locations"])[0], \ "Error: wrong location for file." assert len(outputFile["parents"]) == len(inputLFNs), \ "Error: Output file has wrong number of parents." for outputParent in outputFile["parents"]: assert outputParent["lfn"] in inputLFNs, \ "Error: Unknown parent file: %s" % outputParent["lfn"] fwjrRuns = {} for run in fwkJobReportFile["runs"]: fwjrRuns[run.run] = run.lumis for run in outputFile["runs"]: assert run.run in fwjrRuns, \ "Error: Extra run in output: %s" % run.run for lumi in run: assert lumi in fwjrRuns[run.run], \ "Error: Extra lumi: %s" % lumi fwjrRuns[run.run].remove(lumi) if len(fwjrRuns[run.run]) == 0: del fwjrRuns[run.run] assert len(fwjrRuns) == 0, \ "Error: Missing runs, lumis: %s" % fwjrRuns testJobGroup = JobGroup(id = testJob["jobgroup"]) testJobGroup.loadData() jobGroupFileset = testJobGroup.output jobGroupFileset.loadData() assert outputFile["id"] in jobGroupFileset.getFiles(type = "id"), \ "Error: output file not in jobgroup fileset." if testJob["mask"]["FirstEvent"] == None: assert outputFile["first_event"] == 0, \ "Error: first event not set correctly: 0, %s" % \ outputFile["first_event"] else: assert testJob["mask"]["FirstEvent"] == outputFile["first_event"], \ "Error: last event not set correctly: %s, %s" % \ (testJob["mask"]["FirstEvent"], outputFile["first_event"]) return def testReportHandling(self): """ _testReportHandling_ Verify that we're able to parse a CMSSW report, convert it to a Report() style report, pickle it and then have the accountant process it. """ self.procPath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(self.procPath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("cmsRun1"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fwjrPath = os.path.join(self.tempDir, "ProcReport.pkl") cmsRunStep = myReport.retrieveStep("cmsRun1") cmsRunStep.status = 0 myReport.setTaskName('/TestWF/None') myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testJob["id"], fwjrPath = fwjrPath) pFile = DBSBufferFile(lfn = "/path/to/some/lfn", size = 600000, events = 60000) pFile.setAlgorithm(appName = "cmsRun", appVer = "UNKNOWN", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") pFile.setDatasetPath("/bogus/dataset/path") #pFile.addRun(Run(1, *[45])) pFile.create() config = self.createConfig(workerThreads = 1) accountant = JobAccountantPoller(config) accountant.setup() accountant.algorithm() self.verifyJobSuccess(self.testJob["id"]) self.verifyFileMetaData(self.testJob["id"], myReport.getAllFilesFromStep("cmsRun1")) inputFile = File(lfn = "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR09_R_34X_V5_All_v1/0000/outputRECORECO.root") inputFile.load() self.testMergeJob = Job(name = "testMergeJob", files = [inputFile]) self.testMergeJob.create(group = self.mergeJobGroup) self.testMergeJob["state"] = "complete" self.stateChangeAction.execute(jobs = [self.testMergeJob]) self.mergePath = os.path.join(WMCore.WMBase.getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMergeReport.xml") myReport = Report("mergeReco") myReport.parse(self.mergePath) # Fake some metadata that should be added by the stageout scripts. for fileRef in myReport.getAllFileRefsFromStep("mergeReco"): fileRef.size = 1024 fileRef.location = "cmssrm.fnal.gov" fileRef.dataset = {"applicationName": "cmsRun", "applicationVersion": "CMSSW_3_4_2_patch1", "primaryDataset": "MinimumBias", "processedDataset": "Rereco-v1", "dataTier": "RECO"} fwjrPath = os.path.join(self.tempDir, "MergeReport.pkl") myReport.setTaskName('/MergeWF/None') cmsRunStep = myReport.retrieveStep("mergeReco") cmsRunStep.status = 0 myReport.persist(fwjrPath) self.setFWJRAction.execute(jobID = self.testMergeJob["id"], fwjrPath = fwjrPath) accountant.algorithm() self.verifyJobSuccess(self.testMergeJob["id"]) self.verifyFileMetaData(self.testMergeJob["id"], myReport.getAllFilesFromStep("mergeReco")) return
class PhEDExInjectorSubscriberTest(EmulatedUnitTestCase): """ _PhEDExInjectorSubscriberTest_ Create some database inside DBSBuffer, run the subscriber algorithm using a PhEDEx emulator and verify that it works both in unsafe and safe mode. For unsafe mode there a WMBS database is also created """ def __init__(self, methodName='runTest'): super(PhEDExInjectorSubscriberTest, self).__init__(methodName=methodName) self.blockAName = None self.blockBName = None def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ super(PhEDExInjectorSubscriberTest, self).setUp() self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault=False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() super(PhEDExInjectorSubscriberTest, self).tearDown() def createConfig(self): """ _createConfig_ Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because we are using Emulators """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeDatasets = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 return config def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) insertWorkflow = buffer3Factory(classname="InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) datasetAction = uploadFactory(classname="NewDataset") createAction = uploadFactory(classname="CreateBlocks") datasetAction.execute(datasetPath=self.testDatasetA) datasetAction.execute(datasetPath=self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBufferBlock(name=self.blockAName, location="srm-cms.cern.ch", datasetpath=None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBufferBlock(name=self.blockBName, location="srm-cms.cern.ch", datasetpath=None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks=[newBlockA, newBlockB]) bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname="DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname="NewSubscription") datasetA = DBSBufferDataset(path=self.testDatasetA) datasetB = DBSBufferDataset(path=self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return def testNormalModeSubscriptions(self): """ _testNormalModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on normal operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig() subscriber = PhEDExInjectorPoller(config) subscriber.setup({}) subscriber.subscribeDatasets() phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"]) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[ 0].fetchall() self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed") result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[ 0].fetchall() self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed") # Reset and run again and make sure that no duplicate subscriptions are created myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0") subscriber.subscribeDatasets() self.assertEqual(len(subscriptions[self.testDatasetA]), 3) self.assertEqual(len(subscriptions[self.testDatasetB]), 2) return
class PhEDExInjectorPollerTest(unittest.TestCase): """ _PhEDExInjectorPollerTest_ Unit tests for the PhEDExInjector. Create some database inside DBSBuffer and then have the PhEDExInjector upload the data to PhEDEx. Pull the data back down and verify that everything is complete. """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://cmsweb.cern.ch/phedex/datasvc/json/test" self.dbsURL = "http://vocms09.cern.ch:8880/cms_dbs_int_local_yy_writer/servlet/DBSServlet" self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase = True) self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"], useDefault = False) myThread = threading.currentThread() daofactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = daofactory(classname = "DBSBufferFiles.AddLocation") locationAction.execute(siteName = "srm-cms.cern.ch") locationAction.execute(siteName = "se.fnal.gov") self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/%s/PromptReco-v1/RECO" % makeUUID() self.testDatasetB = "/%s/CRUZET11-v1/RAW" % makeUUID() self.phedex = PhEDEx({"endpoint": self.phedexURL}, "json") return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the datbase. We'll inject files with the location set as an SE name as well as a PhEDEx node name as well. """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) insertWorkflow = buffer3Factory(classname = "InsertWorkflow") insertWorkflow.execute("BogusRequest", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileD.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile(lfn = makeUUID(), size = 1024, events = 10, checksums = checksums, locations = set(["srm-cms.cern.ch"])) testFileE.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8", appFam = "RECO", psetHash = "GIBBERISH", configContent = "MOREGIBBERISH") testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) datasetAction = uploadFactory(classname = "NewDataset") createAction = uploadFactory(classname = "CreateBlocks") datasetAction.execute(datasetPath = self.testDatasetA) datasetAction.execute(datasetPath = self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBufferBlock(name = self.blockAName, location = "srm-cms.cern.ch", datasetpath = None) newBlockA.setDataset(self.testDatasetA, 'data', 'VALID') newBlockA.status = 'Closed' newBlockB = DBSBufferBlock(name = self.blockBName, location = "srm-cms.cern.ch", datasetpath = None) newBlockB.setDataset(self.testDatasetB, 'data', 'VALID') newBlockB.status = 'Closed' createAction.execute(blocks = [newBlockA, newBlockB]) bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = bufferFactory(classname = "DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname = "DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "LOCAL") fileStatus.execute(testFileB["lfn"], "LOCAL") fileStatus.execute(testFileC["lfn"], "LOCAL") fileStatus.execute(testFileD["lfn"], "LOCAL") fileStatus.execute(testFileE["lfn"], "LOCAL") associateWorkflow = buffer3Factory(classname = "DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequest", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequest", "BogusTask") return def createConfig(self): """ _createConfig_ Create a config for the PhEDExInjector with paths to the test DBS and PhEDEx instances. """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeMSS = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 return config def retrieveReplicaInfoForBlock(self, blockName): """ _retrieveReplicaInfoForBlock_ Retrieve the replica information for a block. It takes several minutes after a block is injected for the statistics to be calculated, so this will block until that information is available. """ attempts = 0 while attempts < 15: result = self.phedex.getReplicaInfoForFiles(block = blockName) if "phedex" in result: if "block" in result["phedex"]: if len(result["phedex"]["block"]) != 0: return result["phedex"]["block"][0] attempts += 1 time.sleep(20) logging.info("Could not retrieve replica info for block: %s" % blockName) return None @attr("integration") def testPoller(self): """ _testPoller_ Stuff the database and have the poller upload files to PhEDEx. Retrieve replica information for the uploaded blocks and verify that all files have been injected. """ return self.stuffDatabase() poller = PhEDExInjectorPoller(self.createConfig()) poller.setup(parameters = None) poller.algorithm(parameters = None) replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName) goldenLFNs = [] for file in self.testFilesA: goldenLFNs.append(file["lfn"]) for replicaFile in replicaInfo["file"]: assert replicaFile["name"] in goldenLFNs, \ "Error: Extra file in replica block: %s" % replicaFile["name"] goldenLFNs.remove(replicaFile["name"]) assert len(goldenLFNs) == 0, \ "Error: Files missing from PhEDEx replica: %s" % goldenLFNs replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName) goldenLFNs = [] for file in self.testFilesB: goldenLFNs.append(file["lfn"]) for replicaFile in replicaInfo["file"]: assert replicaFile["name"] in goldenLFNs, \ "Error: Extra file in replica block: %s" % replicaFile["name"] goldenLFNs.remove(replicaFile["name"]) assert len(goldenLFNs) == 0, \ "Error: Files missing from PhEDEx replica: %s" % goldenLFNs myThread = threading.currentThread() daofactory = DAOFactory(package = "WMComponent.DBSUpload.Database", logger = myThread.logger, dbinterface = myThread.dbi) setBlock = daofactory(classname = "SetBlockStatus") setBlock.execute(self.blockAName, locations = None, open_status = "InGlobalDBS") poller.algorithm(parameters = None) replicaInfo = self.retrieveReplicaInfoForBlock(self.blockAName) assert replicaInfo["is_open"] == "n", \ "Error: block should be closed." replicaInfo = self.retrieveReplicaInfoForBlock(self.blockBName) assert replicaInfo["is_open"] == "y", \ "Error: block should be open." return def test_CustodialSiteA(self): """ _CustodialSiteA_ Check the custodialSite stuff by DAO, since I don't have a cert First make sure we properly handle having no custodialSite """ self.stuffDatabase() myThread = threading.currentThread() daofactory = DAOFactory(package = "WMComponent.PhEDExInjector.Database", logger = myThread.logger, dbinterface = myThread.dbi) getUninjected = daofactory(classname = "GetUninjectedFiles") uninjectedFiles = getUninjected.execute() self.assertEqual(uninjectedFiles.keys(), ['srm-cms.cern.ch']) return
class scaleTestFiller: """ _scaleTestFiller_ Initializes the DB and the DBSUploader On __call__() it creates data and uploads it. """ def __init__(self): """ __init__ Init the DB """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer"], useDefault=False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory( package="WMComponent.DBSBuffer.Database", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.bufferFactory( classname="DBSBufferFiles.AddLocation") locationAction.execute(siteName="se1.cern.ch") locationAction.execute(siteName="se1.fnal.gov") locationAction.execute(siteName="malpaquet") config = self.getConfig() self.dbsUploader = DBSUploadPoller(config=config) return def __call__(self): """ __call__ Generate some random data """ # Generate somewhere between one and a thousand files name = "ThisIsATest_%s" % (makeUUID()) nFiles = random.randint(10, 2000) name = name.replace('-', '_') name = '%s-v0' % name files = self.getFiles(name=name, nFiles=nFiles) print "Inserting %i files for dataset %s" % (nFiles * 2, name) try: self.dbsUploader.algorithm() except: self.dbsUploader.close() raise # Repeat just to make sure try: self.dbsUploader.algorithm() except: self.dbsUploader.close() raise return def getConfig(self): """ _getConfig_ This creates the actual config file used by the component """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = 'DBSUpload' config.Agent.useHeartbeat = False #Now the CoreDatabase information #This should be the dialect, dburl, etc config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.component_("DBS3Upload") config.DBS3Upload.pollInterval = 10 config.DBS3Upload.logLevel = 'DEBUG' config.DBS3Upload.DBSBlockMaxFiles = 500 config.DBS3Upload.DBSBlockMaxTime = 600 config.DBS3Upload.DBSBlockMaxSize = 999999999999 config.DBS3Upload.dbsUrl = 'http://cms-xen40.fnal.gov:8787/dbs/prod/global/DBSWriter' config.DBS3Upload.namespace = 'WMComponent.DBS3Buffer.DBSUpload' config.DBS3Upload.componentDir = os.path.join(os.getcwd(), 'Components') config.DBS3Upload.nProcesses = 1 config.DBS3Upload.dbsWaitTime = 1 return config def getFiles(self, name, tier='RECO', nFiles=12, site="malpaquet", nLumis=1): """ Create some quick dummy test files """ files = [] for f in range(nFiles): testFile = DBSBufferFile( lfn='/data/store/random/random/RANDOM/test/0/%s-%s-%i.root' % (name, site, f), size=1024, events=20, checksums={'cksum': 1}) testFile.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFile.setDatasetPath("/%s/%s/%s" % (name, name, tier)) lumis = [] for i in range(nLumis): lumis.append((f * 100000) + i) testFile.addRun(Run(1, *lumis)) testFile.setAcquisitionEra(name.split('-')[0]) testFile.setProcessingVer("0") testFile.setGlobalTag("Weird") testFile.create() testFile.setLocation(site) files.append(testFile) count = 0 for f in files: count += 1 testFileChild = DBSBufferFile( lfn= '/data/store/random/random/RANDOM/test/0/%s-%s-%i-child.root' % (name, site, count), size=1024, events=10, checksums={'cksum': 1}) testFileChild.setAlgorithm(appName=name, appVer="CMSSW_3_1_1", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH") testFileChild.setDatasetPath("/%s/%s_2/RECO" % (name, name)) testFileChild.addRun(Run(1, *[45])) testFileChild.create() testFileChild.setLocation(site) testFileChild.addParents([f['lfn']]) return files
class WorkflowManagerTest(unittest.TestCase): """ TestCase for TestWorkflowManager module """ _maxMessage = 10 def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all needed WMBS tables. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = \ ['WMCore.Agent.Database', 'WMComponent.WorkflowManager.Database', 'WMCore.ThreadPool', 'WMCore.MsgService', 'WMCore.WMBS'], useDefault = False) return def tearDown(self): """ _tearDown_ Database deletion """ self.testInit.clearDatabase() return def getConfig(self): """ _getConfig_ Get defaults WorkflowManager parameters """ return self.testInit.getConfiguration( os.path.join(WMCore.WMInit.getWMBASE(), \ 'src/python/WMComponent/WorkflowManager/DefaultConfig.py')) def testA(self): """ _testA_ Handle AddWorkflowToManage events """ myThread = threading.currentThread() config = self.getConfig() testWorkflowManager = WorkflowManager(config) testWorkflowManager.prepareToStart() for i in xrange(0, WorkflowManagerTest._maxMessage): workflow = Workflow(spec = "testSpec.xml", owner = "riahi", \ name = "testWorkflow" + str(i), task = "testTask") workflow.create() for j in xrange(0, 3): workflowManagerdict = {'payload':{'WorkflowId' : workflow.id \ , 'FilesetMatch': 'FILESET_' + str(j) ,'SplitAlgo':'NO SPLITALGO', 'Type':'NO TYPE' }} testWorkflowManager.handleMessage( \ type = 'AddWorkflowToManage' , payload = workflowManagerdict ) time.sleep(30) myThread.workerThreadManager.terminateWorkers() while threading.activeCount() > 1: print('Currently: '+str(threading.activeCount())+\ ' Threads. Wait until all our threads have finished') time.sleep(1)
class HarvestTest(unittest.TestCase): """ _HarvestTest_ Test for EndOfRun job splitter """ def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN") self.fileset1 = Fileset(name="TestFileset1") for fileNum in range(11): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset=self.fileset1, workflow=workflow1, split_algo="Harvest", type="Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return def tearDown(self): """ _tearDown_ """ self.testInit.clearDatabase() EmulatorSetup.deleteConfig(self.configFile) return def getConfig(self): """ _getConfig_ """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL', None) config.JobStateMachine.couchDBName = 'wmagent_jobdump' return config def finishJobs(self, jobGroups, subscription=None): """ _finishJobs_ """ if not subscription: subscription = self.subscription1 for f in subscription.acquiredFiles(): subscription.completeFiles(f) for jobGroup in jobGroups: self.changer.propagate(jobGroup.jobs, 'executing', 'created') self.changer.propagate(jobGroup.jobs, 'complete', 'executing') self.changer.propagate(jobGroup.jobs, 'success', 'complete') self.changer.propagate(jobGroup.jobs, 'cleanout', 'success') return def testHarvestEndOfRunTrigger(self): """ _testDQMHarvestEndOfRunTrigger_ Make sure that the basic splitting algo works, which is only, ExpressMerge is ALL done, fire a job against that fileset """ self.assertEqual(self.fileset1.open, True, "Fileset is closed. Shouldn't") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 0, "We got 1 or more jobGroups with an open fileset and no periodic configuration") self.fileset1.markOpen(False) self.assertEqual(self.fileset1.open, False, "Fileset is opened, why?") # We should also check if there are aqcuired files, if there are, there are jobs, # we don't want to fire another jobs while previous are running (output is integrating whatever input) # TODO : The above one we can do when all is done. Not priority jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1, "Harvest jobsplitter didn't create a single jobGroup after the fileset was closed") return def testPeriodicTrigger(self): """ _testPeriodicTrigger_ """ self.assertEqual(self.fileset1.open, True, "Fileset is not open, not testing periodic here") # Test timeout (5s for this first test) # there should be no acquired files, if there are, shouldn't be a job # self.subscription1.acquireFiles(self.subscription1.availableFiles().pop()) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=3) self.assertEqual(len(jobGroups), 1, "Didn't created the first periodic job when there were acquired files") # For the whole thing to work, faking the first job finishing, and putting the files as complete self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for fileNum in range(12, 24): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.commit() # Testing that it doesn't create a job unless the delay is past jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 0, "Created one or more job, when there were non-acquired file and the period is not passed by") time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, and there weren't and the period is passed by") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding more of files, so we have new stuff to process for fileNum in range(26, 36): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.commit() # Trying to create another job just afterwards, it shouldn't, because it should respect the configured delay jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 0, "Created one or more job, there are new files, but the delay is not past") time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "Didn't created one or more job, there are new files and the delay is past") # Last check is whether the job gets all the files or not numFilesJob = jobGroups[0].jobs[0].getFiles() numFilesFileset = self.fileset1.getFiles() self.assertEqual(numFilesJob, numFilesFileset, "Job didn't got all the files") # Finishing out previous jobs self.finishJobs(jobGroups) # Adding files for the first location for fileNum in range(38, 48): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.commit() # Then another location for fileNum in range(50, 56): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T2_CH_CERN') self.fileset1.addFile(newFile) self.fileset1.commit() # We should have jobs in both locations time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups[0].getJobs()), 2, "We didn't get 2 jobs for 2 locations") firstJobLocation = jobGroups[0].getJobs()[0].getFileLocations()[0] secondJobLocation = jobGroups[0].getJobs()[1].getFileLocations()[0] self.assertEqual(firstJobLocation, 'T2_CH_CERN') self.assertEqual(secondJobLocation, 'T1_US_FNAL') self.finishJobs(jobGroups) for fileNum in range(60, 65): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(2, *[2])) newFile.setLocation('T2_CH_CERN') self.fileset1.addFile(newFile) self.fileset1.commit() for fileNum in range(70, 75): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(3, *[3])) newFile.setLocation('T2_CH_CERN') self.fileset1.addFile(newFile) self.fileset1.commit() time.sleep(2) jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=self.subscription1) jobGroups = jobFactory(periodic_harvest_interval=2) # This is one of the most "complicated" tests so worth to comment, 4 jobs should be created # 1 - all previous files from SomeSE and run = 1 (a lot, like ~45) # 2 - Few files from SomeSE3, Run = 1 # 3 - Few files from SomeSE3, Run = 2 # 4 - Few files from SomeSE3, Run = 3 self.assertEqual(len(jobGroups[0].getJobs()), 4, "We didn't get 4 jobs for adding 2 different runs to SomeSE3") return def testMultipleRunHarvesting(self): """ _testMultipleRunHarvesting_ Add some files with multiple runs in each, make sure the jobs are created by location and run. Verify each job mask afterwards. Note that in this test run are splitted between sites, in real life that MUST NOT happen we still don't support that. """ multipleFilesFileset = Fileset(name="TestFileset") newFile = File("/some/file/test1", size=1000, events=100) newFile.addRun(Run(1, *[1, 3, 4, 5, 6, 7])) newFile.addRun(Run(2, *[1, 2, 4, 5, 6, 7])) newFile.setLocation('T1_US_FNAL_Disk') multipleFilesFileset.addFile(newFile) newFile = File("/some/file/test2", size=1000, events=100) newFile.addRun(Run(1, *[2, 8])) newFile.addRun(Run(2, *[3, 8])) newFile.setLocation('T2_CH_CERN') multipleFilesFileset.addFile(newFile) multipleFilesFileset.create() harvestingWorkflow = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((str(run), lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) self.finishJobs(jobGroups, harvestSub) newFile = File("/some/file/test3", size=1000, events=100) newFile.addRun(Run(1, *range(9, 15))) newFile.setLocation('T2_CH_CERN') multipleFilesFileset.addFile(newFile) multipleFilesFileset.commit() time.sleep(2) jobGroups = jobFactory(periodic_harvest_interval=2) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) harvestingWorkflowSib = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflowSib", task="TestSib") harvestingWorkflowSib.create() harvestSubSib = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflowSib, split_algo="Harvest", type="Harvesting") harvestSubSib.create() jobFactorySib = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSubSib) multipleFilesFileset.markOpen(False) jobGroups = jobFactorySib(periodic_harvest_sibling=True) self.assertEqual(len(jobGroups), 0, "A single job group was created") self.finishJobs(jobGroups, harvestSub) jobGroups = jobFactorySib(periodic_harvest_sibling=True) self.assertEqual(len(jobGroups), 1, "A single job group was not created") self.assertEqual(len(jobGroups[0].getJobs()), 4, "Four jobs were not created") for job in jobGroups[0].getJobs(): runs = job['mask'].getRunAndLumis() self.assertEqual(len(runs), 1, "Job has more than one run configured") ll = LumiList(compactList={1: [[1, 1], [3, 7], [2, 2], [8, 8], [9, 14]], 2: [[1, 2], [4, 7], [3, 3], [8, 8]]}) run = runs.keys()[0] for lumiPair in runs[run]: for lumi in range(lumiPair[0], lumiPair[1] + 1): self.assertTrue((run, lumi) in ll, "All of %s not in %s" % (lumiPair, ll)) def testMultiRunHarvesting(self): """ _testMultiRunHarvesting_ Provided a fileset with a couple of files and different runs, create a single job for all the runs at a specific location, which also adds a baggage to the job (True) which is later on looked up by SetupCMSSWPSet. """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(dqmHarvestUnit="multiRun") self.assertEqual(len(jobGroups), 1) for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 1) for job in jobGroup.jobs: baggage = job.getBaggage() self.assertTrue(getattr(baggage, "multiRun", False), "It's supposed to be a multiRun job") self.assertEqual(getattr(baggage, "runLimits", ""), "-1-6") def testByRunHarvesting(self): """ _testByRunHarvesting_ Provided a fileset with a couple of files and 4 different runs, create one single job per run and location. The multiRun baggage should be false in this case. """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True, "Fileset should be open!") harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory() self.assertEqual(len(jobGroups), 1, "Should have created 1 job group") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 6, "Should have created 6 jobs") for job in jobGroup.jobs: baggage = job.getBaggage() self.assertFalse(getattr(baggage, "multiRun", False), "It's supposed to be a byRun job") def testByRunAndRunWhitelist(self): """ _testByRunAndRunWhitelist_ Create harvesting jobs by run for the runs provided in the RunWhitelist """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(runWhitelist=[1, 3]) self.assertEqual(len(jobGroups), 1, "One jobgroup per location") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 2) def testByRunAndRunBlacklist(self): """ _testByRunAndRunWhitelist_ Create harvesting jobs by run for the runs provided in the RunWhitelist """ multipleFilesFileset = createCommonFileset() self.assertEqual(multipleFilesFileset.open, True) harvestingWorkflow = Workflow(spec="spec.xml", owner="amaltaro", name="TestWorkflow", task="Test") harvestingWorkflow.create() harvestSub = Subscription(fileset=multipleFilesFileset, workflow=harvestingWorkflow, split_algo="Harvest", type="Harvesting") harvestSub.create() multipleFilesFileset.markOpen(False) self.assertEqual(multipleFilesFileset.open, False, "Fileset should now be closed") jobFactory = self.splitterFactory(package="WMCore.WMBS", subscription=harvestSub) jobGroups = jobFactory(runWhitelist=[1, 2, 3, 4, 5], runBlacklist=[1, 3]) self.assertEqual(len(jobGroups), 1, "One jobgroup per location") for jobGroup in jobGroups: self.assertEqual(len(jobGroup.jobs), 3)
class DBSUploadTest(unittest.TestCase): """ TestCase for DBSUpload module """ def setUp(self): """ _setUp_ setUp function for unittest """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMComponent.DBS3Buffer"], useDefault = False) self.testDir = self.testInit.generateWorkDir(deleteOnDestruction = False) self.configFile = EmulatorSetup.setupWMAgentConfig() myThread = threading.currentThread() self.bufferFactory = DAOFactory(package = "WMComponent.DBS3Buffer", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.bufferFactory(classname = "DBSBufferFiles.AddLocation") locationAction.execute(siteName = "se1.cern.ch") locationAction.execute(siteName = "se1.fnal.gov") locationAction.execute(siteName = "malpaquet") self.dbsUrl = "https://*****:*****@attr("integration") def testBasicUpload(self): """ _testBasicUpload_ Verify that we can successfully upload to DBS3. Also verify that the uploader correctly handles files parentage when uploading. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) # The algorithm needs to be run twice. On the first iteration it will # create all the blocks and upload one. On the second iteration it will # timeout and upload the second block. dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) # Verify the files made it into DBS3. self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Inject some more parent files and some child files into DBSBuffer. # Run the uploader twice, only the parent files should be added to DBS3. (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader another two times to upload the child files. Verify # that the child files were uploaded. dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return @attr("integration") def testDualUpload(self): """ _testDualUpload_ Verify that the dual upload mode works correctly. """ self.dbsApi = DbsApi(url = self.dbsUrl) config = self.getConfig() dbsUploader = DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test verifies that uploader will poll and then not do anything # as the database is empty. dbsUploader.algorithm() acqEra = "Summer%s" % (int(time.time())) parentFiles = self.createParentFiles(acqEra) (moreParentFiles, childFiles) = \ self.createFilesWithChildren(parentFiles, acqEra) allFiles = parentFiles + moreParentFiles allBlocks = [] for i in range(4): DBSBufferDataset(parentFiles[0]["datasetPath"]).create() blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "Open" dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in allFiles[i * 5 : (i * 5) + 5]: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) if i < 2: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUtil.updateFileStatus([dbsBlock], "InDBS") allBlocks.append(dbsBlock) DBSBufferDataset(childFiles[0]["datasetPath"]).create() blockName = childFiles[0]["datasetPath"] + "#" + makeUUID() dbsBlock = DBSBufferBlock(blockName, location = "malpaquet", datasetpath = None) dbsBlock.status = "InDBS" dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID') dbsUtil.createBlocks([dbsBlock]) for file in childFiles: dbsBlock.addFile(file, 'data', 'VALID') dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]}) dbsUtil.updateFileStatus([dbsBlock], "InDBS") dbsUploader.algorithm() time.sleep(5) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles) # Change the status of the rest of the parent blocks so we can upload # them and the children. for dbsBlock in allBlocks: dbsBlock.status = "InDBS" dbsUtil.updateBlocks([dbsBlock]) dbsUploader.algorithm() time.sleep(5) self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles) # Run the uploader one more time to upload the children. dbsUploader.algorithm() time.sleep(5) self.verifyData(childFiles[0]["datasetPath"], childFiles) return def testCloseSettingsPerWorkflow(self): """ _testCloseSettingsPerWorkflow_ Test the block closing mechanics in the DBS3 uploader, this uses a fake dbs api to avoid reliance on external services. """ # Signal trapExit that we are a friend os.environ["DONT_TRAP_EXIT"] = "True" try: # Monkey patch the imports of DbsApi from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller MockDBSUploadPoller.DbsApi = MockDbsApi # Set the poller and the dbsUtil for verification myThread = threading.currentThread() (_, dbsFilePath) = mkstemp(dir = self.testDir) self.dbsUrl = dbsFilePath config = self.getConfig() dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config) dbsUtil = DBSBufferUtil() # First test is event based limits and timeout with no new files. # Set the files and workflow acqEra = "TropicalSeason%s" % (int(time.time())) workflowName = 'TestWorkload%s' % (int(time.time())) taskPath = '/%s/TestProcessing' % workflowName self.injectWorkflow(workflowName, taskPath, MaxWaitTime = 2, MaxFiles = 100, MaxEvents = 150) self.createParentFiles(acqEra, nFiles = 20, workflowName = workflowName, taskPath = taskPath) # The algorithm needs to be run twice. On the first iteration it will # create all the blocks and upload one with less than 150 events. # On the second iteration the second block is uploaded. dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 1) globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall() notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall() self.assertEqual(len(globalFiles), 14) self.assertEqual(len(notUploadedFiles), 6) # Check the fake DBS for data fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 2) for block in fakeDBSInfo: self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['file_count'], 7) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) time.sleep(3) dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 0) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 3) for block in fakeDBSInfo: if block['block']['file_count'] != 6: self.assertEqual(block['block']['file_count'], 7) self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) # Now check the limit by size and timeout with new files acqEra = "TropicalSeason%s" % (int(time.time())) workflowName = 'TestWorkload%s' % (int(time.time())) taskPath = '/%s/TestProcessing' % workflowName self.injectWorkflow(workflowName, taskPath, MaxWaitTime = 2, MaxFiles = 5, MaxEvents = 200000000) self.createParentFiles(acqEra, nFiles = 16, workflowName = workflowName, taskPath = taskPath) dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 1) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 6) for block in fakeDBSInfo: if acqEra in block['block']['block_name']: self.assertEqual(block['block']['file_count'], 5) self.assertTrue('block_events' not in block['block']) self.assertTrue('close_settings' not in block) self.assertEqual(block['block']['open_for_writing'], 0) # Put more files, they will go into the same block and then it will be closed # after timeout time.sleep(3) self.createParentFiles(acqEra, nFiles = 3, workflowName = workflowName, taskPath = taskPath) dbsUploader.algorithm() dbsUploader.checkBlocks() openBlocks = dbsUtil.findOpenBlocks() self.assertEqual(len(openBlocks), 0) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 7) for block in fakeDBSInfo: if acqEra in block['block']['block_name']: if block['block']['file_count'] < 5: self.assertEqual(block['block']['file_count'], 4) else: self.assertEqual(block['block']['file_count'], 5) self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) # Finally test size limits acqEra = "TropicalSeason%s" % (int(time.time())) workflowName = 'TestWorkload%s' % (int(time.time())) taskPath = '/%s/TestProcessing' % workflowName self.injectWorkflow(workflowName, taskPath, MaxWaitTime = 1, MaxFiles = 500, MaxEvents = 200000000, MaxSize = 2048) self.createParentFiles(acqEra, nFiles = 7, workflowName = workflowName, taskPath = taskPath) dbsUploader.algorithm() dbsUploader.checkBlocks() time.sleep(2) dbsUploader.algorithm() dbsUploader.checkBlocks() self.assertEqual(len(openBlocks), 0) fakeDBS = open(self.dbsUrl, 'r') fakeDBSInfo = json.load(fakeDBS) fakeDBS.close() self.assertEqual(len(fakeDBSInfo), 11) for block in fakeDBSInfo: if acqEra in block['block']['block_name']: if block['block']['file_count'] != 1: self.assertEqual(block['block']['block_size'], 2048) self.assertEqual(block['block']['file_count'], 2) self.assertTrue('block_events' not in block['block']) self.assertEqual(block['block']['open_for_writing'], 0) self.assertTrue('close_settings' not in block) except: self.fail("We failed at some point in the test") finally: # We don't trust anyone else with _exit del os.environ["DONT_TRAP_EXIT"] return
class PhEDExInjectorSubscriberTest(unittest.TestCase): """ _PhEDExInjectorSubscriberTest_ Create some database inside DBSBuffer, run the subscriber algorithm using a PhEDEx emulator and verify that it works both in unsafe and safe mode. For unsafe mode there a WMBS database is also created """ def setUp(self): """ _setUp_ Install the DBSBuffer schema into the database and connect to PhEDEx. """ self.phedexURL = "https://bogus.cern.ch/bogus" self.dbsURL = "https://bogus.cern.ch/bogus" EmulatorHelper.setEmulators(phedex=True, dbs=True, siteDB=True) self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"], useDefault=False) self.testFilesA = [] self.testFilesB = [] self.testDatasetA = "/BogusPrimary/Run2012Z-PromptReco-v1/RECO" self.testDatasetB = "/BogusPrimary/CRUZET11-v1/RAW" return def tearDown(self): """ _tearDown_ Delete the database. """ self.testInit.clearDatabase() EmulatorHelper.resetEmulators() def createConfig(self): """ _createConfig_ Create a config for the PhEDExInjector, paths to DBS and PhEDEx are dummies because we are using Emulators """ config = self.testInit.getConfiguration() config.component_("DBSInterface") config.DBSInterface.globalDBSUrl = self.dbsURL config.component_("PhEDExInjector") config.PhEDExInjector.phedexurl = self.phedexURL config.PhEDExInjector.subscribeDatasets = True config.PhEDExInjector.group = "Saturn" config.PhEDExInjector.pollInterval = 30 config.PhEDExInjector.subscribeInterval = 60 return config def stuffDatabase(self): """ _stuffDatabase_ Fill the dbsbuffer with some files and blocks. We'll insert a total of 5 files spanning two blocks. There will be a total of two datasets inserted into the database. All files will be already in GLOBAL and in_phedex """ myThread = threading.currentThread() buffer3Factory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) insertWorkflow = buffer3Factory(classname="InsertWorkflow") insertWorkflow.execute("BogusRequestA", "BogusTask", 0, 0, 0, 0) insertWorkflow.execute("BogusRequestB", "BogusTask", 0, 0, 0, 0) checksums = {"adler32": "1234", "cksum": "5678"} testFileA = DBSBufferFile( lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"]) ) testFileA.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileA.setDatasetPath(self.testDatasetA) testFileA.addRun(Run(2, *[45])) testFileA.create() testFileB = DBSBufferFile( lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"]) ) testFileB.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileB.setDatasetPath(self.testDatasetA) testFileB.addRun(Run(2, *[45])) testFileB.create() testFileC = DBSBufferFile( lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"]) ) testFileC.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileC.setDatasetPath(self.testDatasetA) testFileC.addRun(Run(2, *[45])) testFileC.create() self.testFilesA.append(testFileA) self.testFilesA.append(testFileB) self.testFilesA.append(testFileC) testFileD = DBSBufferFile( lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"]) ) testFileD.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileD.setDatasetPath(self.testDatasetB) testFileD.addRun(Run(2, *[45])) testFileD.create() testFileE = DBSBufferFile( lfn=makeUUID(), size=1024, events=10, checksums=checksums, locations=set(["srm-cms.cern.ch"]) ) testFileE.setAlgorithm( appName="cmsRun", appVer="CMSSW_2_1_8", appFam="RECO", psetHash="GIBBERISH", configContent="MOREGIBBERISH" ) testFileE.setDatasetPath(self.testDatasetB) testFileE.addRun(Run(2, *[45])) testFileE.create() self.testFilesB.append(testFileD) self.testFilesB.append(testFileE) uploadFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) datasetAction = uploadFactory(classname="NewDataset") createAction = uploadFactory(classname="CreateBlocks") datasetAction.execute(datasetPath=self.testDatasetA) datasetAction.execute(datasetPath=self.testDatasetB) self.blockAName = self.testDatasetA + "#" + makeUUID() self.blockBName = self.testDatasetB + "#" + makeUUID() newBlockA = DBSBufferBlock(name=self.blockAName, location="srm-cms.cern.ch", datasetpath=None) newBlockA.setDataset(self.testDatasetA, "data", "VALID") newBlockA.status = "Closed" newBlockB = DBSBufferBlock(name=self.blockBName, location="srm-cms.cern.ch", datasetpath=None) newBlockB.setDataset(self.testDatasetB, "data", "VALID") newBlockB.status = "Closed" createAction.execute(blocks=[newBlockA, newBlockB]) bufferFactory = DAOFactory(package="WMComponent.DBS3Buffer", logger=myThread.logger, dbinterface=myThread.dbi) setBlock = bufferFactory(classname="DBSBufferFiles.SetBlock") setBlock.execute(testFileA["lfn"], self.blockAName) setBlock.execute(testFileB["lfn"], self.blockAName) setBlock.execute(testFileC["lfn"], self.blockAName) setBlock.execute(testFileD["lfn"], self.blockBName) setBlock.execute(testFileE["lfn"], self.blockBName) fileStatus = bufferFactory(classname="DBSBufferFiles.SetStatus") fileStatus.execute(testFileA["lfn"], "GLOBAL") fileStatus.execute(testFileB["lfn"], "GLOBAL") fileStatus.execute(testFileC["lfn"], "GLOBAL") fileStatus.execute(testFileD["lfn"], "GLOBAL") fileStatus.execute(testFileE["lfn"], "GLOBAL") phedexStatus = bufferFactory(classname="DBSBufferFiles.SetPhEDExStatus") phedexStatus.execute(testFileA["lfn"], 1) phedexStatus.execute(testFileB["lfn"], 1) phedexStatus.execute(testFileC["lfn"], 1) phedexStatus.execute(testFileD["lfn"], 1) phedexStatus.execute(testFileE["lfn"], 1) associateWorkflow = buffer3Factory(classname="DBSBufferFiles.AssociateWorkflowToFile") associateWorkflow.execute(testFileA["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileB["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileC["lfn"], "BogusRequestA", "BogusTask") associateWorkflow.execute(testFileD["lfn"], "BogusRequestB", "BogusTask") associateWorkflow.execute(testFileE["lfn"], "BogusRequestB", "BogusTask") # Make the desired subscriptions insertSubAction = buffer3Factory(classname="NewSubscription") datasetA = DBSBufferDataset(path=self.testDatasetA) datasetB = DBSBufferDataset(path=self.testDatasetB) workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl")) insertSubAction.execute(datasetA.exists(), workload.getSubscriptionInformation()[self.testDatasetA]) insertSubAction.execute(datasetB.exists(), workload.getSubscriptionInformation()[self.testDatasetB]) return def testNormalModeSubscriptions(self): """ _testNormalModeSubscriptions_ Tests that we can make custodial/non-custodial subscriptions on normal operation mode, this time we don't need WMBS for anything. All is subscribed in one go. Check that the requests are correct. """ self.stuffDatabase() config = self.createConfig() subscriber = PhEDExInjectorPoller(config) subscriber.setup({}) subscriber.subscribeDatasets() phedexInstance = subscriber.phedex subscriptions = phedexInstance.subRequests # Let's check /BogusPrimary/Run2012Z-PromptReco-v1/RECO # According to the spec, this should be custodial at T1_US_FNAL # Non-custodial at T1_UK_RAL and T3_CO_Uniandes # Autoapproved in all sites # Priority is normal self.assertTrue(self.testDatasetA in subscriptions, "Dataset A was not subscribed") subInfoA = subscriptions[self.testDatasetA] self.assertEqual(len(subInfoA), 3, "Dataset A was not subscribed to all sites") for subInfo in subInfoA: site = subInfo["node"] self.assertEqual(subInfo["priority"], "normal", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T3_CO_Uniandes": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual( subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"] ) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset A at %s" % subInfo["node"]) elif site == "T1_US_FNAL_MSS": self.assertEqual(subInfo["custodial"], "y", "Wrong custodiality for dataset A at %s" % subInfo["node"]) self.assertEqual( subInfo["request_only"], "n", "Wrong requestOnly for dataset A at %s" % subInfo["node"] ) self.assertEqual(subInfo["move"], "y", "Wrong subscription type for dataset A at %s" % subInfo["node"]) else: self.fail("Dataset A was subscribed to a wrong site %s" % site) # Now check /BogusPrimary/CRUZET11-v1/RAW # According to the spec, this is not custodial anywhere # Non-custodial at T1_UK_RAL and T2_CH_CERN # Request only at both sites and with high priority self.assertTrue(self.testDatasetB in subscriptions, "Dataset B was not subscribed") subInfoB = subscriptions[self.testDatasetB] self.assertEqual(len(subInfoB), 2, "Dataset B was not subscribed to all sites") for subInfo in subInfoB: site = subInfo["node"] self.assertEqual(subInfo["priority"], "high", "Wrong priority for subscription") if site == "T1_UK_RAL_MSS" or site == "T2_CH_CERN": self.assertEqual(subInfo["custodial"], "n", "Wrong custodiality for dataset B at %s" % subInfo["node"]) self.assertEqual( subInfo["request_only"], "y", "Wrong requestOnly for dataset B at %s" % subInfo["node"] ) self.assertEqual(subInfo["move"], "n", "Wrong subscription type for dataset B at %s" % subInfo["node"]) else: self.fail("Dataset B was subscribed to a wrong site %s" % site) myThread = threading.currentThread() result = myThread.dbi.processData("SELECT COUNT(*) FROM dbsbuffer_dataset_subscription where subscribed = 1")[ 0 ].fetchall() self.assertEqual(result[0][0], 5, "Not all datasets were marked as subscribed") result = myThread.dbi.processData("SELECT site FROM dbsbuffer_dataset_subscription where subscribed = 0")[ 0 ].fetchall() self.assertEqual(result[0][0], "T1_IT_CNAF", "A non-valid CMS site was subscribed") # Reset and run again and make sure that no duplicate subscriptions are created myThread.dbi.processData("UPDATE dbsbuffer_dataset_subscription SET subscribed = 0") subscriber.subscribeDatasets() self.assertEqual(len(subscriptions[self.testDatasetA]), 3) self.assertEqual(len(subscriptions[self.testDatasetB]), 2) return
class HarnessTest(unittest.TestCase): """ TestCase for TestComponent module """ tempDir = None def setUp(self): """ setup for test. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema() def tearDown(self): """ Delete database """ self.testInit.clearDatabase() def testB(self): raise nose.SkipTest config = self.testInit.getConfiguration() self.tempDir = self.testInit.generateWorkDir(config) config.component_("TestComponent") config.TestComponent.logLevel = 'INFO' config.section_("General") config.TestComponent.componentDir = os.path.join( \ self.tempDir, "Components/TestComponent1") config.General.workDir = config.TestComponent.componentDir os.makedirs(config.TestComponent.componentDir) # as this is a test we build the string from our global environment # parameters normally you put this straight into the DefaultConfig.py file: # testInit.getConfiguration returns from the environment variable by default testComponent = TestComponent(config) testComponent.prepareToStart() testComponent.handleMessage('LogState', '') testComponent.handleMessage('TestMessage1', 'TestMessag1Payload') testComponent.handleMessage('TestMessage2', 'TestMessag2Payload') testComponent.handleMessage('TestMessage3', 'TestMessag3Payload') testComponent.handleMessage('TestMessage4', 'TestMessag4Payload') testComponent.handleMessage('Logging.DEBUG', '') testComponent.handleMessage('Logging.WARNING', '') testComponent.handleMessage('Logging.CRITICAL', '') testComponent.handleMessage('Logging.ERROR', '') testComponent.handleMessage('Logging.INFO', '') testComponent.handleMessage('Logging.SQLDEBUG', '') testComponent.handleMessage('TestComponent:Logging.DEBUG', '') testComponent.handleMessage('TestComponent:Logging.WARNING', '') testComponent.handleMessage('TestComponent:Logging.CRITICAL', '') testComponent.handleMessage('TestComponent:Logging.ERROR', '') testComponent.handleMessage('TestComponent:Logging.INFO', '') testComponent.handleMessage('TestComponent:Logging.SQLDEBUG', '') # test a non existing message (to generate an error) errorMsg = '' try: testComponent.handleMessage('NonExistingMessageType', '') except Exception, ex: errorMsg = str(ex) self.assertTrue( errorMsg.startswith('Message NonExistingMessageType with payload'))