def testTestNonProxySplitting(self): """ _TestNonProxySplitting_ Test and see if we can split things without a proxy. """ config = self.getConfig() config.JobCreator.workerThreads = 1 name = makeUUID() workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.stuffWMBS(workflowURL=workloadPath, name=name) testJobCreator = JobCreatorPoller(config=config) testJobCreator.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), 1) result = getJobsAction.execute(state='Created', jobType="Merge") self.assertEqual(len(result), 0) return
def testE_TestNonProxySplitting(self): """ _TestNonProxySplitting_ Test and see if we can split things without a proxy. """ myThread = threading.currentThread() config = self.getConfig() config.JobCreator.workerThreads = 1 name = makeUUID() workloadName = 'TestWorkload' workload = self.createWorkload(workloadName = workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.stuffWMBS(workflowURL = workloadPath, name = name) testJobCreator = JobCreatorPoller(config = config) testJobCreator.algorithm() getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 1) result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), 0) return
def testD_HugeTest(self): """ Don't run this one either """ return myThread = threading.currentThread() config = self.getConfig() name = makeUUID() nSubs = 10 nFiles = 5000 workloadName = 'Tier1ReReco' workload = self.createWorkload(workloadName = workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name = name, nSubs = nSubs, nFiles = nFiles, workflowURL = workloadPath) testJobCreator = JobCreatorPoller(config = config) # First, can we run once without everything crashing? startTime = time.time() testJobCreator.algorithm() stopTime = time.time() getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), nSubs*nFiles) print("Job took %f seconds to run" %(stopTime - startTime)) # Count database objects result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) return
def testProfileWorker(self): """ Profile where the work actually gets done You shouldn't be running this one either, since it doesn't test anything. """ name = makeUUID() nSubs = 5 nFiles = 500 workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) config = self.getConfig() configDict = {"couchURL": config.JobStateMachine.couchurl, "couchDBName": config.JobStateMachine.couchDBName, 'jobCacheDir': config.JobCreator.jobCacheDir, 'defaultJobType': config.JobCreator.defaultJobType} subs = [{"subscription": 1}, {"subscription": 2}, {"subscription": 3}, {"subscription": 4}, {"subscription": 5}] testJobCreator = JobCreatorPoller(**configDict) cProfile.runctx("testJobCreator.algorithm(parameters = input)", globals(), locals(), filename="workStats.stat") p = pstats.Stats('workStats.stat') p.sort_stats('cumulative') p.print_stats(.2) return
def testProfilePoller(self): """ Profile your performance You shouldn't be running this normally because it doesn't do anything """ name = makeUUID() nSubs = 5 nFiles = 1500 workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) config = self.getConfig() testJobCreator = JobCreatorPoller(config=config) cProfile.runctx("testJobCreator.algorithm()", globals(), locals(), filename="testStats.stat") getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") time.sleep(10) self.assertEqual(len(result), nSubs * nFiles) p = pstats.Stats('testStats.stat') p.sort_stats('cumulative') p.print_stats(.2) return
def testE_TestNonProxySplitting(self): """ _TestNonProxySplitting_ Test and see if we can split things without a proxy. """ myThread = threading.currentThread() config = self.getConfig() config.JobCreator.workerThreads = 1 name = makeUUID() workloadName = 'TestWorkload' workload = self.createWorkload(workloadName = workloadName) # Change the file splitting algo procTask = workload.getTask("ReReco") procTask.setSplittingAlgorithm("ParentlessMergeBySize", min_merge_size = 1, max_merge_size = 100000, max_merge_events = 200000) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.stuffWMBS(workflowURL = workloadPath, name = name) testJobCreator = JobCreatorPoller(config = config) testJobCreator.algorithm() getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 1) result = getJobsAction.execute(state = 'Created', jobType = "Merge") self.assertEqual(len(result), 0) return
def preInitialization(self): """ Step that actually adds the worker thread properly """ print("JobCreator.preInitialization") # Add event loop to worker manager myThread = threading.currentThread() pollInterval = self.config.JobCreator.pollInterval logging.info("Setting poll interval to %s seconds", pollInterval) myThread.workerThreadManager.addWorker(JobCreatorPoller(self.config), pollInterval) return
def testA_StraightThrough(self): """ _StraightThrough_ Just run everything straight through without any variations """ # Do pre-submit job check nRunning = getCondorRunningJobs() self.assertEqual(nRunning, 0, "User currently has %i running jobs. Test will not continue" % (nRunning)) myThread = threading.currentThread() workload = self.createTestWorkload() config = self.getConfig() name = 'WMAgent_Test1' site = self.sites[0] nSubs = 5 nFiles = 10 workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') # Create a collection of files self.createFileCollection(name = name, nSubs = nSubs, nFiles = nFiles, workflowURL = workloadPath, site = site) ############################################################ # Test the JobCreator config.Agent.componentName = 'JobCreator' testJobCreator = JobCreatorPoller(config = config) testJobCreator.algorithm() time.sleep(5) # Did all jobs get created? getJobsAction = self.daoFactory(classname = "Jobs.GetAllJobs") result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), nSubs*nFiles) # Count database objects result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, 'TestWorkload', 'ReReco') self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory)) self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0') # First job should be in here self.assertTrue('job_1' in os.listdir(groupDirectory)) jobFile = os.path.join(groupDirectory, 'job_1', 'job.pkl') self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, 'r') job = cPickle.load(f) f.close() self.assertEqual(job['workflow'], name) self.assertEqual(len(job['input_files']), 1) self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2') ############################################################### # Now test the JobSubmitter config.Agent.componentName = 'JobSubmitter' testJobSubmitter = JobSubmitterPoller(config = config) testJobSubmitter.algorithm() # Check that jobs are in the right state result = getJobsAction.execute(state = 'Created', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) # Check assigned locations getLocationAction = self.daoFactory(classname = "Jobs.GetLocation") for id in result: loc = getLocationAction.execute(jobid = id) self.assertEqual(loc, [[site]]) # Check to make sure we have running jobs nRunning = getCondorRunningJobs() self.assertEqual(nRunning, nFiles * nSubs) ################################################################# # Now the JobTracker config.Agent.componentName = 'JobTracker' testJobTracker = JobTrackerPoller(config = config) testJobTracker.setup() testJobTracker.algorithm() # Running the algo without removing the jobs should do nothing result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) condorRM() time.sleep(1) # All jobs gone? nRunning = getCondorRunningJobs() self.assertEqual(nRunning, 0) testJobTracker.algorithm() time.sleep(5) # Running the algo without removing the jobs should do nothing result = getJobsAction.execute(state = 'Executing', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) ################################################################# # Now the JobAccountant # First you need to load all jobs self.getFWJRAction = self.daoFactory(classname = "Jobs.GetFWJRByState") completeJobs = self.getFWJRAction.execute(state = "complete") # Create reports for all jobs self.createReports(jobs = completeJobs, retryCount = 0) config.Agent.componentName = 'JobAccountant' testJobAccountant = JobAccountantPoller(config = config) testJobAccountant.setup() # It should do something with the jobs testJobAccountant.algorithm() # All the jobs should be done now result = getJobsAction.execute(state = 'Complete', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Success', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) ####################################################################### # Now the JobArchiver config.Agent.componentName = 'JobArchiver' testJobArchiver = JobArchiverPoller(config = config) testJobArchiver.algorithm() # All the jobs should be cleaned up result = getJobsAction.execute(state = 'Success', jobType = "Processing") self.assertEqual(len(result), 0) result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing") self.assertEqual(len(result), nSubs * nFiles) logDir = os.path.join(self.testDir, 'logs') for job in completeJobs: self.assertFalse(os.path.exists(job['fwjr_path'])) jobFolder = 'JobCluster_%i' \ % (int(job['id']/config.JobArchiver.numberOfJobsToCluster)) jobPath = os.path.join(logDir, jobFolder, 'Job_%i.tar' %(job['id'])) self.assertTrue(os.path.isfile(jobPath)) self.assertTrue(os.path.getsize(jobPath) > 0) ########################################################################### # Now the TaskAchiver config.Agent.componentName = 'TaskArchiver' testTaskArchiver = TaskArchiverPoller(config = config) testTaskArchiver.algorithm() result = getJobsAction.execute(state = 'Cleanout', jobType = "Processing") self.assertEqual(len(result), 0) for jdict in completeJobs: job = Job(id = jdict['id']) self.assertFalse(job.exists()) if os.path.isdir('testDir'): shutil.rmtree('testDir') shutil.copytree('%s' %self.testDir, os.path.join(os.getcwd(), 'testDir')) return
def testVerySimpleTest(self): """ _VerySimpleTest_ Just test that everything works...more or less """ # return myThread = threading.currentThread() config = self.getConfig() name = makeUUID() nSubs = 5 nFiles = 10 workloadName = 'TestWorkload' dummyWorkload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) testJobCreator = JobCreatorPoller(config=config) # First, can we run once without everything crashing? testJobCreator.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) # Count database objects result = myThread.dbi.processData('SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, 'jobCacheDir', 'TestWorkload', 'ReReco') # It should have at least one jobGroup self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory)) # But no more then twenty self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0') # First job should be in here listOfDirs = [] for tmpDirectory in os.listdir(testDirectory): listOfDirs.extend(os.listdir(os.path.join(testDirectory, tmpDirectory))) self.assertTrue('job_1' in listOfDirs) self.assertTrue('job_2' in listOfDirs) self.assertTrue('job_3' in listOfDirs) jobDir = os.listdir(groupDirectory)[0] jobFile = os.path.join(groupDirectory, jobDir, 'job.pkl') self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, 'r') job = pickle.load(f) f.close() self.assertEqual(job.baggage.PresetSeeder.generator.initialSeed, 1001) self.assertEqual(job.baggage.PresetSeeder.evtgenproducer.initialSeed, 1001) self.assertEqual(job['workflow'], name) self.assertEqual(len(job['input_files']), 1) self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2') return
def testA_VerySimpleTest(self): """ _VerySimpleTest_ Just test that everything works...more or less """ #return myThread = threading.currentThread() config = self.getConfig() name = makeUUID() nSubs = 5 nFiles = 10 workloadName = 'TestWorkload' workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, 'workloadTest', 'TestWorkload', 'WMSandbox', 'WMWorkload.pkl') self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) testJobCreator = JobCreatorPoller(config=config) # First, can we run once without everything crashing? testJobCreator.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state='Created', jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) # Count database objects result = myThread.dbi.processData( 'SELECT * FROM wmbs_sub_files_acquired')[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, 'jobCacheDir', 'TestWorkload', 'ReReco') # It should have at least one jobGroup self.assertTrue('JobCollection_1_0' in os.listdir(testDirectory)) # But no more then twenty self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, 'JobCollection_1_0') # First job should be in here listOfDirs = [] for tmpDirectory in os.listdir(testDirectory): listOfDirs.extend( os.listdir(os.path.join(testDirectory, tmpDirectory))) self.assertTrue('job_1' in listOfDirs) self.assertTrue('job_2' in listOfDirs) self.assertTrue('job_3' in listOfDirs) jobDir = os.listdir(groupDirectory)[0] jobFile = os.path.join(groupDirectory, jobDir, 'job.pkl') self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, 'r') job = cPickle.load(f) f.close() self.assertEqual(job.baggage.PresetSeeder.generator.initialSeed, 1001) self.assertEqual(job.baggage.PresetSeeder.evtgenproducer.initialSeed, 1001) self.assertEqual(job['workflow'], name) self.assertEqual(len(job['input_files']), 1) self.assertEqual(os.path.basename(job['sandbox']), 'TestWorkload-Sandbox.tar.bz2') return
def testA_VerySimpleTest(self): """ _VerySimpleTest_ Just test that everything works...more or less """ # return myThread = threading.currentThread() config = self.getConfig() name = makeUUID() nSubs = 5 nFiles = 10 workloadName = "TestWorkload" workload = self.createWorkload(workloadName=workloadName) workloadPath = os.path.join(self.testDir, "workloadTest", "TestWorkload", "WMSandbox", "WMWorkload.pkl") self.createJobCollection(name=name, nSubs=nSubs, nFiles=nFiles, workflowURL=workloadPath) testJobCreator = JobCreatorPoller(config=config) # First, can we run once without everything crashing? testJobCreator.algorithm() getJobsAction = self.daoFactory(classname="Jobs.GetAllJobs") result = getJobsAction.execute(state="Created", jobType="Processing") self.assertEqual(len(result), nSubs * nFiles) # Count database objects result = myThread.dbi.processData("SELECT * FROM wmbs_sub_files_acquired")[0].fetchall() self.assertEqual(len(result), nSubs * nFiles) # Find the test directory testDirectory = os.path.join(self.testDir, "jobCacheDir", "TestWorkload", "ReReco") # It should have at least one jobGroup self.assertTrue("JobCollection_1_0" in os.listdir(testDirectory)) # But no more then twenty self.assertTrue(len(os.listdir(testDirectory)) <= 20) groupDirectory = os.path.join(testDirectory, "JobCollection_1_0") # First job should be in here listOfDirs = [] for tmpDirectory in os.listdir(testDirectory): listOfDirs.extend(os.listdir(os.path.join(testDirectory, tmpDirectory))) self.assertTrue("job_1" in listOfDirs) self.assertTrue("job_2" in listOfDirs) self.assertTrue("job_3" in listOfDirs) jobDir = os.listdir(groupDirectory)[0] jobFile = os.path.join(groupDirectory, jobDir, "job.pkl") self.assertTrue(os.path.isfile(jobFile)) f = open(jobFile, "r") job = cPickle.load(f) f.close() self.assertEqual(job["workflow"], name) self.assertEqual(len(job["input_files"]), 1) self.assertEqual(os.path.basename(job["sandbox"]), "TestWorkload-Sandbox.tar.bz2") self.assertEqual(job["priority"], 1) return