def testUpdateJobSlots(self): """ _testUpdateJobSlots_ Verify that it is possible to update the number of job slots at a site. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1") siteInfo = myResourceControl.listSiteInfo("testSite1") self.assertEqual(siteInfo["pending_slots"], 10, "Error: Pending slots is wrong.") self.assertEqual(siteInfo["running_slots"], 20, "Error: Running slots is wrong.") myResourceControl.setJobSlotsForSite("testSite1", pendingJobSlots=20) siteInfo = myResourceControl.listSiteInfo("testSite1") self.assertEqual(siteInfo["pending_slots"], 20, "Error: Pending slots is wrong.") myResourceControl.setJobSlotsForSite("testSite1", runningJobSlots=40) siteInfo = myResourceControl.listSiteInfo("testSite1") self.assertEqual(siteInfo["running_slots"], 40, "Error: Running slots is wrong.") myResourceControl.setJobSlotsForSite("testSite1", 5, 10) siteInfo = myResourceControl.listSiteInfo("testSite1") self.assertEqual(siteInfo["pending_slots"], 5, "Error: Pending slots is wrong.") self.assertEqual(siteInfo["running_slots"], 10, "Error: Running slots is wrong.") return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. """ myThread = threading.currentThread() self.sites = ['T2_US_Florida', 'T2_US_UCSD', 'T2_TW_Taiwan', 'T1_CH_CERN'] self.testInit = TestInit(__file__) self.testInit.setLogging(logLevel = logging.DEBUG) self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'], useDefault = False) self.testInit.setupCouch("dashboardreporter_t/jobs", "JobDump") self.testInit.setupCouch("dashboardreporter_t/fwjrs", "FWJRDump") resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, seName = site, ceName = site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000) self.testDir = self.testInit.generateWorkDir() self.alertsReceiver = None return
def testAbortedState(self): """ _testAbortedState_ Check that we can kill jobs when a site is set to aborted ### We no longer need this test as we are not killing jobs that are running """ self.tempDir = self.testInit.generateWorkDir() config = self.createConfig() myResourceControl = ResourceControl(config) myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "MockPlugin") myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T1_IT_CNAF", "MockPlugin") myResourceControl.insertThreshold("testSite1", "Processing", 20, 10) myResourceControl.insertThreshold("testSite1", "Merge", 200, 100) myResourceControl.insertThreshold("testSite2", "Processing", 50, 25) myResourceControl.insertThreshold("testSite2", "Merge", 135, 65) self.createJobs() myResourceControl.changeSiteState("testSite1", "Aborted") ## Now check the tempDir for a FWJR for the killed job reportPath = os.path.join(self.tempDir, "Report.0.pkl") report = Report() report.load(reportPath) self.assertEqual(report.getExitCode(), 71301) return
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") self.myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=self.myThread.dbi) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 1000, 2000, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 1000, 2000, "T2_CH_CERN", "T2_CH_CERN") self.performanceParams = {'timePerEvent': 12, 'memoryRequirement': 2300, 'sizePerEvent': 400} # dummy workflow self.testWorkflow = Workflow(spec="spec.xml", owner="dmwm", name="testWorkflow", task="Test") self.testWorkflow.create() return
def testListSiteInfo(self): """ _testListSiteInfo_ Verify that the listSiteInfo() methods works properly. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1") myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2") siteInfo = myResourceControl.listSiteInfo("testSite1") self.assertEqual(siteInfo["site_name"], "testSite1", "Error: Site name is wrong.") self.assertEqual(siteInfo["pnn"], ["testSE1"], "Error: SE name is wrong.") self.assertEqual(siteInfo["ce_name"], "testCE1", "Error: CE name is wrong.") self.assertEqual(siteInfo["pending_slots"], 10, "Error: Pending slots is wrong.") self.assertEqual(siteInfo["running_slots"], 20, "Error: Pending slots is wrong.") return
def setResourceThresholds(self, site, **options): """ _setResourceThresholds_ Utility to set resource thresholds """ if not options: options = {'state' : 'Normal', 'runningSlots' : 10, 'pendingSlots' : 5, 'tasks' : ['Processing', 'Merge'], 'Processing' : {'pendingSlots' : 5, 'runningSlots' : 10}, 'Merge' : {'pendingSlots' : 2, 'runningSlots' : 5}} resourceControl = ResourceControl() resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site, plugin = "MockPlugin", pendingSlots = options['pendingSlots'], runningSlots = options['runningSlots'], cmsName = site) for task in options['tasks']: resourceControl.insertThreshold(siteName = site, taskType = task, maxSlots = options[task]['runningSlots'], pendingSlots = options[task]['pendingSlots']) if options.get('state'): resourceControl.changeSiteState(site, options.get('state')) return
def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"], useDefault = False) self.daoFactory = DAOFactory(package = "WMCore.BossAir", logger = myThread.logger, dbinterface = myThread.dbi) resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'Xanadu', seName = 'se.Xanadu', ceName = 'Xanadu', plugin = "TestPlugin") resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) # Create user wmbsFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) newuser = wmbsFactory(classname = "Users.New") newuser.execute(dn = "mnorman", group_name = "phgroup", role_name = "cmsrole")
def setUp(self): """ _setUp_ Set everything up. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault = False) self.testInit.setupCouch("jobsubmittercaching_t/jobs", "JobDump") self.testInit.setupCouch("jobsubmittercaching_t/fwjrs", "FWJRDump") resourceControl = ResourceControl() for siteName in ["T1_US_FNAL", "T1_UK_RAL"]: resourceControl.insertSite(siteName = siteName, pnn = "se.%s" % (siteName), ceName = siteName, plugin = "CondorPlugin", cmsName = siteName) resourceControl.insertThreshold(siteName = siteName, taskType = "Processing", maxSlots = 10000, pendingSlots = 10000) self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig() return
def testThresholdsForSite(self): """ _testThresholdsForSite_ Check that we can get the thresholds in intelligible form for each site """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10) myResourceControl.insertThreshold("testSite1", "Merge", 5) result = myResourceControl.thresholdBySite(siteName="testSite1") procInfo = {} mergInfo = {} for res in result: if res["task_type"] == "Processing": procInfo = res elif res["task_type"] == "Merge": mergInfo = res self.assertEqual(procInfo.get("job_slots", None), 20) self.assertEqual(procInfo.get("max_slots", None), 10) self.assertEqual(mergInfo.get("job_slots", None), 20) self.assertEqual(mergInfo.get("max_slots", None), 5) return
def testThresholdsForSite(self): """ _testThresholdsForSite_ Check that we can get the thresholds in intelligible form for each site """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 8) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3) result = myResourceControl.thresholdBySite(siteName="testSite1") procInfo = {} mergInfo = {} for res in result: if res['task_type'] == 'Processing': procInfo = res elif res['task_type'] == 'Merge': mergInfo = res self.assertEqual(procInfo.get('pending_slots', None), 20) self.assertEqual(procInfo.get('running_slots', None), 40) self.assertEqual(procInfo.get('max_slots', None), 10) self.assertEqual(procInfo.get('task_pending_slots', None), 8) self.assertEqual(mergInfo.get('pending_slots', None), 20) self.assertEqual(mergInfo.get('running_slots', None), 40) self.assertEqual(mergInfo.get('max_slots', None), 5) self.assertEqual(mergInfo.get('task_pending_slots', None), 3) return
def testThresholdPriority(self): """ _testThresholdPriority_ Test that we get things back in priority order """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 8) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3) # test default task priorities result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 4) self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 0) myResourceControl.changeTaskPriority("Merge", 3) myResourceControl.changeTaskPriority("Processing", 1) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 3) self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 1) myResourceControl.changeTaskPriority("Merge", 1) myResourceControl.changeTaskPriority("Processing", 3) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds']['Merge']['priority'], 1) self.assertEqual(result['testSite1']['thresholds']['Processing']['priority'], 3) return
def setUp(self): """ _setUp_ Setup the database and logging connection. Try to create all of the WMBS tables. Also, create some dummy locations. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules = ['WMCore.WMBS', 'WMCore.ResourceControl', 'WMCore.Agent.Database'], useDefault = False) self.couchdbname = "jobcreator_t" self.testInit.setupCouch("%s/jobs" % self.couchdbname, "JobDump") self.testInit.setupCouch("%s/fwjrs" % self.couchdbname, "FWJRDump") myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daoFactory(classname = "Locations.New") for site in self.sites: locationAction.execute(siteName = site, seName = site) #Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, seName = site, ceName = site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) self.resourceControl = resourceControl self._setup = True self._teardown = False self.testDir = self.testInit.generateWorkDir() self.cwd = os.getcwd() # Set heartbeat self.componentName = 'JobCreator' self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() return
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.tearDown() self.testInit.setSchema( customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"], useDefault=False) self.testInit.setupCouch("bossair_t/jobs", "JobDump") self.testInit.setupCouch("bossair_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs") # Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName=site, pnn='%s_PNN' % site, cmsName=site, ceName=site, plugin="SimpleCondorPlugin", pendingSlots=1000, runningSlots=2000) resourceControl.insertThreshold(siteName=site, taskType='Processing', maxSlots=1000, pendingSlots=1000) site = 'T3_US_Xanadu' resourceControl.insertSite(siteName=site, pnn='%s_PNN' % site, cmsName=site, ceName=site, plugin="TestPlugin") resourceControl.insertThreshold(siteName=site, taskType='Processing', maxSlots=10000, pendingSlots=10000) # Create user newuser = self.daoFactory(classname="Users.New") newuser.execute(dn="tapas", group_name="phgroup", role_name="cmsrole") # We actually need the user name self.user = getpass.getuser() # Change this to the working dir to keep track of error and log files from condor self.testDir = self.testInit.generateWorkDir() # Set heartbeat componentName = 'test' self.heartbeatAPI = HeartbeatAPI(componentName) self.heartbeatAPI.registerComponent() componentName = 'JobTracker' self.heartbeatAPI2 = HeartbeatAPI(componentName) self.heartbeatAPI2.registerComponent() return
def setUp(self): """ _setUp_ Set up vital components """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS",'WMCore.MsgService', 'WMCore.ResourceControl', 'WMCore.ThreadPool', 'WMCore.Agent.Database'], useDefault = False) myThread = threading.currentThread() self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = self.daoFactory(classname = "Locations.New") pendingSlots = self.daoFactory(classname = "Locations.SetPendingSlots") for site in self.sites: locationAction.execute(siteName = site, seName = 'se.%s' % (site), ceName = site) pendingSlots.execute(siteName = site, pendingSlots = 1000) #Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) self.testDir = self.testInit.generateWorkDir() # Set heartbeat for component in self.components: heartbeatAPI = HeartbeatAPI(component) heartbeatAPI.registerComponent() return
def testDrain(self): """Drain a site""" myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, priority=1) myResourceControl.drainSite("testSite1") result = myResourceControl.listThresholdsForCreate() self.assertTrue(result["testSite1"]["drain"]) # re-renable myResourceControl.drainSite("testSite1", drain=False) result = myResourceControl.listThresholdsForCreate() self.assertFalse(result["testSite1"]["drain"])
def setUp(self): """ Standard setup: Now with 100% more couch """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema( customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"], useDefault=False, ) self.testInit.setupCouch("jobsubmitter_t/jobs", "JobDump") self.testInit.setupCouch("jobsubmitter_t/fwjrs", "FWJRDump") myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = self.daoFactory(classname="Locations.New") locationSlots = self.daoFactory(classname="Locations.SetJobSlots") # We actually need the user name self.user = getpass.getuser() self.ceName = "127.0.0.1" # Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite( siteName=site, seName="se.%s" % (site), ceName=site, plugin="CondorPlugin", pendingSlots=10000, runningSlots=20000, cmsName=site, ) resourceControl.insertThreshold(siteName=site, taskType="Processing", maxSlots=10000) self.testDir = self.testInit.generateWorkDir() # Set heartbeat self.componentName = "JobSubmitter" self.heartbeatAPI = HeartbeatAPI(self.componentName) self.heartbeatAPI.registerComponent() return
def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules = ["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package = "WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package = "WMCore.WMBS", logger = logging, dbinterface = myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("SomeSite", 10, 20, "SomeSE", "SomeCE") myResourceControl.insertSite("SomeSite2", 10, 20, "SomeSE2", "SomeCE2") self.fileset1 = Fileset(name = "TestFileset1") for file in range(11): newFile = File("/some/file/name%d" % file, size = 1000, events = 100) newFile.addRun(Run(1,*[1])) newFile.setLocation('SomeSE') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec = "spec.xml", owner = "hufnagel", name = "TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset = self.fileset1, workflow = workflow1, split_algo = "Harvest", type = "Harvesting") self.subscription1.create() return
def testChangeState(self): """ _testChangeState_ Check that we can change the state between different values and retrieve it through the threshold methods """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 5) result = myResourceControl.listThresholdsForCreate() self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state') myResourceControl.changeSiteState("testSite1", "Down") result = myResourceControl.listThresholdsForCreate() self.assertEqual(result['testSite1']['state'], 'Down', 'Error: Wrong site state')
def testThresholdPriority(self): """ _testThresholdPriority_ Test that we get things back in priority order """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 8) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3) myResourceControl.changeTaskPriority("Merge", 3) myResourceControl.changeTaskPriority("Processing", 1) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Processing') myResourceControl.insertThreshold("testSite1", "Processing", 10, 8) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3) myResourceControl.changeTaskPriority("Merge", 1) myResourceControl.changeTaskPriority("Processing", 3) # Should now be in reverse order result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Processing') myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2") myResourceControl.insertThreshold("testSite2", "Processing", 10, 8) myResourceControl.insertThreshold("testSite2", "Merge", 5, 3) # Should be in the same order for site 1 and 2 result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite2']['thresholds'][0]['task_type'], result['testSite1']['thresholds'][0]['task_type']) self.assertEqual(result['testSite2']['thresholds'][1]['task_type'], result['testSite1']['thresholds'][1]['task_type']) myResourceControl.changeTaskPriority("Merge", 4) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite2']['thresholds'][0]['priority'], 4) return
def testThresholdPriority(self): """ _testThresholdPriority_ Test that we get things back in priority order """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 8, priority = 1) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3, priority = 2) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Processing') myResourceControl.insertThreshold("testSite1", "Processing", 10, 8, priority = 2) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3, priority = 1) # Should now be in reverse order result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Processing') myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2") myResourceControl.insertThreshold("testSite2", "Processing", 10, 8, priority = 1) myResourceControl.insertThreshold("testSite2", "Merge", 5, 3, priority = 2) # Should be in proper order for site 2 result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite2']['thresholds'][0]['task_type'], 'Merge') self.assertEqual(result['testSite2']['thresholds'][1]['task_type'], 'Processing') # Should now be in reverse order for site 1 self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Processing') myResourceControl.insertThreshold("testSite2", "Merge", 20, 10) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite2']['thresholds'][0]['priority'], 2) return
def testChangeSiteState(self): """ _testNewState_ Check that we can change the state between different values and retrieve it through the threshold methods """ self.tempDir = self.testInit.generateWorkDir() config = self.createConfig() myResourceControl = ResourceControl(config) myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 5) result = myResourceControl.listThresholdsForCreate() self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state') myResourceControl.changeSiteState("testSite1", "Down") result = myResourceControl.listThresholdsForCreate() self.assertEqual(result['testSite1']['state'], 'Down', 'Error: Wrong site state')
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"]) self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault = False) self.testInit.setupCouch("jobtracker_t/jobs", "JobDump") self.testInit.setupCouch("jobtracker_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs") #Create sites in resourceControl resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'malpaquet', pnn = 'se.malpaquet', ceName = 'malpaquet', plugin = "CondorPlugin") resourceControl.insertThreshold(siteName = 'malpaquet', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) locationAction = self.daoFactory(classname = "Locations.New") locationAction.execute(siteName = "malpaquet", pnn = "malpaquet", ceName = "malpaquet", plugin = "CondorPlugin") # Create user newuser = self.daoFactory(classname = "Users.New") newuser.execute(dn = "jchurchill") # We actually need the user name self.user = getpass.getuser() self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig()
def setUp(self): """ setup for test. """ super(JobTrackerTest, self).setUp() myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() # self.testInit.clearDatabase(modules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"]) self.testInit.setSchema(customModules=["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl"], useDefault=False) self.testInit.setupCouch("jobtracker_t/jobs", "JobDump") self.testInit.setupCouch("jobtracker_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs") # Create sites in resourceControl resourceControl = ResourceControl() resourceControl.insertSite(siteName='malpaquet', pnn='se.malpaquet', ceName='malpaquet', plugin="CondorPlugin") resourceControl.insertThreshold(siteName='malpaquet', taskType='Processing', \ maxSlots=10000, pendingSlots=10000) locationAction = self.daoFactory(classname="Locations.New") locationAction.execute(siteName="malpaquet", pnn="malpaquet", ceName="malpaquet", plugin="CondorPlugin") # Create user newuser = self.daoFactory(classname="Users.New") newuser.execute(dn="jchurchill") # We actually need the user name self.user = getpass.getuser() self.testDir = self.testInit.generateWorkDir() self.configFile = EmulatorSetup.setupWMAgentConfig()
def setResourceThresholds(self, site, **options): """ _setResourceThresholds_ Utility to set resource thresholds """ if not options: options = { 'state': 'Normal', 'runningSlots': 10, 'pendingSlots': 5, 'tasks': ['Processing', 'Merge'], 'Processing': { 'pendingSlots': 5, 'runningSlots': 10 }, 'Merge': { 'pendingSlots': 2, 'runningSlots': 5, 'priority': 5 } } resourceControl = ResourceControl() resourceControl.insertSite(siteName=site, seName='se.%s' % (site), ceName=site, plugin="MockPlugin", pendingSlots=options['pendingSlots'], runningSlots=options['runningSlots'], cmsName=site) for task in options['tasks']: resourceControl.insertThreshold( siteName=site, taskType=task, maxSlots=options[task]['runningSlots'], pendingSlots=options[task]['pendingSlots'], priority=options[task].get('priority', 1)) if options.get('state'): resourceControl.changeSiteState(site, options.get('state')) return
def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"], useDefault = False) self.daoFactory = DAOFactory(package = "WMCore.BossAir", logger = myThread.logger, dbinterface = myThread.dbi) resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'Xanadu', seName = 'se.Xanadu', ceName = 'Xanadu', plugin = "TestPlugin") resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \ maxSlots = 10000)
def setUp(self): """ _setUp_ Create two subscriptions: One that contains a single file and one that contains multiple files. """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") self.myThread = threading.currentThread() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=self.myThread.dbi) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 1000, 2000, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 1000, 2000, "T2_CH_CERN", "T2_CH_CERN") self.performanceParams = { 'timePerEvent': 12, 'memoryRequirement': 2300, 'sizePerEvent': 400 } # dummy workflow self.testWorkflow = Workflow(spec="spec.xml", owner="dmwm", name="testWorkflow", task="Test") self.testWorkflow.create() if PY3: self.assertItemsEqual = self.assertCountEqual return
def testListSiteInfo(self): """ _testListSiteInfo_ Verify that the listSiteInfo() methods works properly. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, "testSE1", "testCE1") myResourceControl.insertSite("testSite2", 100, "testSE2", "testCE2") siteInfo = myResourceControl.listSiteInfo("testSite1") assert siteInfo["site_name"] == "testSite1", \ "Error: Site name is wrong." assert siteInfo["se_name"] == "testSE1", \ "Error: SE name is wrong." assert siteInfo["ce_name"] == "testCE1", \ "Error: CE name is wrong." assert siteInfo["job_slots"] == 10, \ "Error: Job slots is wrong." return
def testThresholdPriority(self): """ _testThresholdPriority_ Test that we get things back in priority order """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 8) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3) # test default task priorities result = myResourceControl.listThresholdsForSubmit() self.assertEqual( result['testSite1']['thresholds']['Merge']['priority'], 4) self.assertEqual( result['testSite1']['thresholds']['Processing']['priority'], 0) myResourceControl.changeTaskPriority("Merge", 3) myResourceControl.changeTaskPriority("Processing", 1) result = myResourceControl.listThresholdsForSubmit() self.assertEqual( result['testSite1']['thresholds']['Merge']['priority'], 3) self.assertEqual( result['testSite1']['thresholds']['Processing']['priority'], 1) myResourceControl.changeTaskPriority("Merge", 1) myResourceControl.changeTaskPriority("Processing", 3) result = myResourceControl.listThresholdsForSubmit() self.assertEqual( result['testSite1']['thresholds']['Merge']['priority'], 1) self.assertEqual( result['testSite1']['thresholds']['Processing']['priority'], 3) return
def testChangeState(self): """ _testChangeState_ Check that we can change the state between different values and retrieve it through the threshold methods """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 5, priority=1) result = myResourceControl.listThresholdsForCreate() self.assertEqual(result['testSite1']['state'], 'Normal', 'Error: Wrong site state') myResourceControl.changeSiteState("testSite1", "Down") result = myResourceControl.listThresholdsForCreate() self.assertEqual(result['testSite1']['state'], 'Down', 'Error: Wrong site state')
def setResourceThresholds(self, site, **options): """ _setResourceThresholds_ Utility to set resource thresholds """ if not options: options = { "state": "Normal", "runningSlots": 10, "pendingSlots": 5, "tasks": ["Processing", "Merge"], "Processing": {"pendingSlots": 5, "runningSlots": 10}, "Merge": {"pendingSlots": 2, "runningSlots": 5}, } resourceControl = ResourceControl() resourceControl.insertSite( siteName=site, pnn="se.%s" % (site), ceName=site, plugin="MockPlugin", pendingSlots=options["pendingSlots"], runningSlots=options["runningSlots"], cmsName=site, ) for task in options["tasks"]: resourceControl.insertThreshold( siteName=site, taskType=task, maxSlots=options[task]["runningSlots"], pendingSlots=options[task]["pendingSlots"], ) if options.get("state"): resourceControl.changeSiteState(site, options.get("state")) return
def setUp(self): """ _setUp_ """ self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.testInit.setSchema(customModules=["WMCore.WMBS"]) self.splitterFactory = SplitterFactory(package="WMCore.JobSplitting") myThread = threading.currentThread() self.myThread = myThread daoFactory = DAOFactory(package="WMCore.WMBS", logger=logging, dbinterface=myThread.dbi) self.WMBSFactory = daoFactory config = self.getConfig() self.changer = ChangeState(config) myResourceControl = ResourceControl() myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T1_US_FNAL_Disk", "T1_US_FNAL") myResourceControl.insertSite("T1_US_FNAL", 10, 20, "T3_US_FNALLPC", "T1_US_FNAL") myResourceControl.insertSite("T2_CH_CERN", 10, 20, "T2_CH_CERN", "T2_CH_CERN") self.fileset1 = Fileset(name="TestFileset1") for fileNum in range(11): newFile = File("/some/file/name%d" % fileNum, size=1000, events=100) newFile.addRun(Run(1, *[1])) newFile.setLocation('T1_US_FNAL_Disk') self.fileset1.addFile(newFile) self.fileset1.create() workflow1 = Workflow(spec="spec.xml", owner="hufnagel", name="TestWorkflow1", task="Test") workflow1.create() self.subscription1 = Subscription(fileset=self.fileset1, workflow=workflow1, split_algo="Harvest", type="Harvesting") self.subscription1.create() self.configFile = EmulatorSetup.setupWMAgentConfig() return
def setUp(self): myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database" ], useDefault=False) self.daoFactory = DAOFactory(package="WMCore.BossAir", logger=myThread.logger, dbinterface=myThread.dbi) resourceControl = ResourceControl() resourceControl.insertSite(siteName='Xanadu', pnn='se.Xanadu', ceName='Xanadu', plugin="TestPlugin") resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) # Create user wmbsFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) newuser = wmbsFactory(classname="Users.New") newuser.execute(dn="mnorman", group_name="phgroup", role_name="cmsrole") if PY3: self.assertItemsEqual = self.assertCountEqual
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() #self.tearDown() self.testInit.setSchema(customModules = ["WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database"], useDefault = False) self.testInit.setupCouch("bossair_t/jobs", "JobDump") self.testInit.setupCouch("bossair_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs") locationAction = self.daoFactory(classname = "Locations.New") locationSlots = self.daoFactory(classname = "Locations.SetJobSlots") #Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName = site, seName = 'se.%s' % (site), ceName = site, plugin = "CondorPlugin", jobSlots = 1000) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 1000) resourceControl.insertSite(siteName = 'Xanadu', seName = 'se.Xanadu', ceName = 'Xanadu', plugin = "TestPlugin") resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \ maxSlots = 10000) resourceControl.insertSite(siteName = 'jade-cms.hip.fi', seName = 'madhatter.csc.fi', ceName = 'jade-cms.hip.fi', plugin = "ARCPlugin") resourceControl.insertThreshold(siteName = 'jade-cms.hip.fi', taskType = 'Processing', \ maxSlots = 100) # using this for glite submissions resourceControl.insertSite(siteName = 'grid-ce-01.ba.infn.it', seName = 'storm-se-01.ba.infn.it', ceName = 'grid-ce-01.ba.infn.it', plugin = 'gLitePlugin') resourceControl.insertThreshold(siteName = 'grid-ce-01.ba.infn.it', taskType = 'Processing', \ maxSlots = 50) # Create user newuser = self.daoFactory(classname = "Users.New") newuser.execute(dn = "mnorman", group_name = "phgroup", role_name = "cmsrole") # We actually need the user name self.user = getpass.getuser() self.testDir = self.testInit.generateWorkDir() # Set heartbeat componentName = 'test' self.heartbeatAPI = HeartbeatAPI(componentName) self.heartbeatAPI.registerComponent() componentName = 'JobTracker' self.heartbeatAPI2 = HeartbeatAPI(componentName) self.heartbeatAPI2.registerComponent() return
def testCreateSubscription(self): """ _testCreateSubscription_ Verify that the subscription creation code works correctly. """ resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'site1', seName = 'goodse.cern.ch', ceName = 'site1', plugin = "TestPlugin") resourceControl.insertSite(siteName = 'site2', seName = 'goodse2.cern.ch', ceName = 'site2', plugin = "TestPlugin") testWorkload = self.createTestWMSpec() testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock", cachepath = self.workDir) testWMBSHelper.createSubscription() procWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask") procWorkflow.load() self.assertEqual(procWorkflow.owner, "sfoulkes", "Error: Wrong owner: %s" % procWorkflow.owner) self.assertEqual(procWorkflow.group, "DMWM", "Error: Wrong group: %s" % procWorkflow.group) self.assertEqual(procWorkflow.wfType, "TestReReco", "Error: Wrong type.") self.assertEqual(procWorkflow.spec, os.path.join(self.workDir, procWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(procWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") mergedProcOutput = procWorkflow.outputMap["OutputA"][0]["merged_output_fileset"] unmergedProcOutput = procWorkflow.outputMap["OutputA"][0]["output_fileset"] mergedProcOutput.loadData() unmergedProcOutput.loadData() self.assertEqual(mergedProcOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-Merged", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedProcOutput.name, "/TestWorkload/ProcessingTask/unmerged-OutputA", "Error: Unmerged output fileset is wrong.") mergeWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap.keys()), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] unmergedMergeOutput.loadData() self.assertEqual(unmergedMergeOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-Merged", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow(name = "TestWorkload", task = "/TestWorkload/ProcessingTask/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual(mergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual(unmergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Unmerged output fileset is wrong.") self.assertEqual(mergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name = "TestWorkload-ProcessingTask-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset = topLevelFileset, workflow = procWorkflow) procSubscription.loadData() self.assertEqual(len(procSubscription.getWhiteBlackList()), 2, "Error: Wrong site white/black list for proc sub.") for site in procSubscription.getWhiteBlackList(): if site["site_name"] == "site1": self.assertEqual(site["valid"], 1, "Error: Site should be white listed.") else: self.assertEqual(site["valid"], 0, "Error: Site should be black listed.") self.assertEqual(procSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") mergeSubscription = Subscription(fileset = unmergedProcOutput, workflow = mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset = unmergedMergeOutput, workflow = skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return
def testList(self): """ _testList_ Test the functions that list thresholds for creating jobs and submitting jobs. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "LsfPlugin") myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T3_US_FNAL", "LsfPlugin") myResourceControl.insertThreshold("testSite1", "Processing", 20, 10) myResourceControl.insertThreshold("testSite1", "Merge", 200, 100) myResourceControl.insertThreshold("testSite2", "Processing", 50, 25) myResourceControl.insertThreshold("testSite2", "Merge", 135, 65) self.createJobs() createThresholds = myResourceControl.listThresholdsForCreate() submitThresholds = myResourceControl.listThresholdsForSubmit() self.assertEqual(len(createThresholds.keys()), 2, "Error: Wrong number of sites in create thresholds") self.assertEqual(createThresholds["testSite1"]["total_slots"], 10, "Error: Wrong number of slots for site 1") self.assertEqual(createThresholds["testSite2"]["total_slots"], 20, "Error: Wrong number of slots for site 2") # We should have two running jobs with locations at site one, # two running jobs without locations at site two, and one running # job without a location at site one and two. self.assertEqual(createThresholds["testSite1"]["pending_jobs"], {0: 4}, "Error: Wrong number of pending jobs for site 1") # We should have one running job with a location at site 2 and # another running job without a location. self.assertEqual(createThresholds["testSite2"]["pending_jobs"], {0: 2}, "Error: Wrong number of pending jobs for site 2") # We should also have a phedex_name self.assertEqual(createThresholds["testSite1"]["cms_name"], "T1_US_FNAL") self.assertEqual(createThresholds["testSite2"]["cms_name"], "T3_US_FNAL") mergeThreshold1 = None mergeThreshold2 = None procThreshold1 = None procThreshold2 = None self.assertEqual(set(submitThresholds.keys()), set(["testSite1", "testSite2"])) for taskType, threshold in submitThresholds["testSite1"]["thresholds"].items(): if taskType == "Merge": mergeThreshold1 = threshold elif taskType == "Processing": procThreshold1 = threshold for taskType, threshold in submitThresholds["testSite2"]["thresholds"].items(): if taskType == "Merge": mergeThreshold2 = threshold elif taskType == "Processing": procThreshold2 = threshold self.assertEqual(submitThresholds["testSite1"]["total_running_jobs"], 0, "Error: Wrong number of running jobs for submit thresholds.") self.assertEqual(submitThresholds["testSite2"]["total_running_jobs"], 1, "Error: Wrong number of running jobs for submit thresholds.") self.assertEqual(submitThresholds["testSite1"]["total_pending_jobs"], 1, "Error: Wrong number of pending jobs for submit thresholds.") self.assertEqual(submitThresholds["testSite2"]["total_pending_jobs"], 0, "Error: Wrong number of pending jobs for submit thresholds.") self.assertEqual(mergeThreshold1["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(mergeThreshold1["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold1["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold1["task_pending_jobs"], 1, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(mergeThreshold2["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(mergeThreshold2["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold2["task_running_jobs"], 1, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold2["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") return
def testInsert(self): """ _testInsert_ Verify that inserting sites and thresholds works correctly, even if the site or threshold already exists. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1") myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1") myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2") myResourceControl.insertThreshold("testSite1", "Processing", 20, 10) myResourceControl.insertThreshold("testSite1", "Merge", 200, 100) myResourceControl.insertThreshold("testSite1", "Merge", 250, 150) myResourceControl.insertThreshold("testSite2", "Processing", 50, 30) myResourceControl.insertThreshold("testSite2", "Merge", 135, 100) createThresholds = myResourceControl.listThresholdsForCreate() self.assertEqual(len(createThresholds.keys()), 2, "Error: Wrong number of site in Resource Control DB") self.assertTrue("testSite1" in createThresholds.keys(), "Error: Test Site 1 missing from thresholds.") self.assertTrue("testSite2" in createThresholds.keys(), "Error: Test Site 2 missing from thresholds.") self.assertEqual(createThresholds["testSite1"]["total_slots"], 10, "Error: Wrong number of total slots.") self.assertEqual(createThresholds["testSite1"]["pending_jobs"], {0: 0}, "Error: Wrong number of running jobs: %s" % createThresholds["testSite1"]["pending_jobs"]) self.assertEqual(createThresholds["testSite2"]["total_slots"], 100, "Error: Wrong number of total slots.") self.assertEqual(createThresholds["testSite2"]["pending_jobs"], {0: 0}, "Error: Wrong number of running jobs.") thresholds = myResourceControl.listThresholdsForSubmit() self.assertEqual(len(thresholds.keys()), 2, "Error: Wrong number of sites in Resource Control DB") self.assertTrue("testSite1" in thresholds.keys(), "Error: testSite1 missing from thresholds.") self.assertTrue("testSite2" in thresholds.keys(), "Error: testSite2 missing from thresholds.") site1Info = thresholds["testSite1"] site2Info = thresholds["testSite2"] site1Thresholds = site1Info["thresholds"] site2Thresholds = site2Info["thresholds"] procThreshold1 = None procThreshold2 = None mergeThreshold1 = None mergeThreshold2 = None for taskType, threshold in site1Thresholds.items(): if taskType == "Merge": mergeThreshold1 = threshold elif taskType == "Processing": procThreshold1 = threshold for taskType, threshold in site2Thresholds.items(): if taskType == "Merge": mergeThreshold2 = threshold elif taskType == "Processing": procThreshold2 = threshold self.assertEqual(len(site1Thresholds), 2, "Error: Wrong number of task types.") self.assertEqual(len(site2Thresholds), 2, "Error: Wrong number of task types.") self.assertNotEqual(procThreshold1, None) self.assertNotEqual(procThreshold2, None) self.assertNotEqual(mergeThreshold1, None) self.assertNotEqual(mergeThreshold2, None) self.assertEqual(site1Info["total_pending_slots"], 10, "Error: Site thresholds wrong") self.assertEqual(site1Info["total_running_slots"], 20, "Error: Site thresholds wrong") self.assertEqual(site1Info["total_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(site1Info["total_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["max_slots"], 20, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["pending_slots"], 10, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["max_slots"], 250, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["pending_slots"], 150, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_pending_slots"], 100, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_running_slots"], 200, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["max_slots"], 50, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["pending_slots"], 30, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["max_slots"], 135, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["pending_slots"], 100, "Error: Site thresholds wrong")
def setupForKillTest(self, baAPI=None): """ _setupForKillTest_ Inject a workflow into WMBS that has a processing task, a merge task and a cleanup task. Inject files into the various tasks at various processing states (acquired, complete, available...). Also create jobs for each subscription in various states. """ myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) locationAction = daoFactory(classname="Locations.New") changeStateAction = daoFactory(classname="Jobs.ChangeState") resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', seName='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertThreshold(siteName = 'site1', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) userDN = 'someDN' userAction = daoFactory(classname="Users.New") userAction.execute(dn=userDN, group_name='DEFAULT', role_name='DEFAULT') inputFileset = Fileset("input") inputFileset.create() inputFileA = File("lfnA", locations="goodse.cern.ch") inputFileB = File("lfnB", locations="goodse.cern.ch") inputFileC = File("lfnC", locations="goodse.cern.ch") inputFileA.create() inputFileB.create() inputFileC.create() inputFileset.addFile(inputFileA) inputFileset.addFile(inputFileB) inputFileset.addFile(inputFileC) inputFileset.commit() unmergedOutputFileset = Fileset("unmerged") unmergedOutputFileset.create() unmergedFileA = File("ulfnA", locations="goodse.cern.ch") unmergedFileB = File("ulfnB", locations="goodse.cern.ch") unmergedFileC = File("ulfnC", locations="goodse.cern.ch") unmergedFileA.create() unmergedFileB.create() unmergedFileC.create() unmergedOutputFileset.addFile(unmergedFileA) unmergedOutputFileset.addFile(unmergedFileB) unmergedOutputFileset.addFile(unmergedFileC) unmergedOutputFileset.commit() mainProcWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="Proc") mainProcWorkflow.create() mainProcMergeWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="ProcMerge") mainProcMergeWorkflow.create() mainCleanupWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="Cleanup") mainCleanupWorkflow.create() self.mainProcSub = Subscription(fileset=inputFileset, workflow=mainProcWorkflow, type="Processing") self.mainProcSub.create() self.mainProcSub.acquireFiles(inputFileA) self.mainProcSub.completeFiles(inputFileB) procJobGroup = JobGroup(subscription=self.mainProcSub) procJobGroup.create() self.procJobA = Job(name="ProcJobA") self.procJobA["state"] = "new" self.procJobA["location"] = "site1" self.procJobB = Job(name="ProcJobB") self.procJobB["state"] = "executing" self.procJobB["location"] = "site1" self.procJobC = Job(name="ProcJobC") self.procJobC["state"] = "complete" self.procJobC["location"] = "site1" self.procJobA.create(procJobGroup) self.procJobB.create(procJobGroup) self.procJobC.create(procJobGroup) self.mainMergeSub = Subscription(fileset=unmergedOutputFileset, workflow=mainProcMergeWorkflow, type="Merge") self.mainMergeSub.create() self.mainMergeSub.acquireFiles(unmergedFileA) self.mainMergeSub.failFiles(unmergedFileB) mergeJobGroup = JobGroup(subscription=self.mainMergeSub) mergeJobGroup.create() self.mergeJobA = Job(name="MergeJobA") self.mergeJobA["state"] = "exhausted" self.mergeJobA["location"] = "site1" self.mergeJobB = Job(name="MergeJobB") self.mergeJobB["state"] = "cleanout" self.mergeJobB["location"] = "site1" self.mergeJobC = Job(name="MergeJobC") self.mergeJobC["state"] = "new" self.mergeJobC["location"] = "site1" self.mergeJobA.create(mergeJobGroup) self.mergeJobB.create(mergeJobGroup) self.mergeJobC.create(mergeJobGroup) self.mainCleanupSub = Subscription(fileset=unmergedOutputFileset, workflow=mainCleanupWorkflow, type="Cleanup") self.mainCleanupSub.create() self.mainCleanupSub.acquireFiles(unmergedFileA) self.mainCleanupSub.completeFiles(unmergedFileB) cleanupJobGroup = JobGroup(subscription=self.mainCleanupSub) cleanupJobGroup.create() self.cleanupJobA = Job(name="CleanupJobA") self.cleanupJobA["state"] = "new" self.cleanupJobA["location"] = "site1" self.cleanupJobB = Job(name="CleanupJobB") self.cleanupJobB["state"] = "executing" self.cleanupJobB["location"] = "site1" self.cleanupJobC = Job(name="CleanupJobC") self.cleanupJobC["state"] = "complete" self.cleanupJobC["location"] = "site1" self.cleanupJobA.create(cleanupJobGroup) self.cleanupJobB.create(cleanupJobGroup) self.cleanupJobC.create(cleanupJobGroup) jobList = [ self.procJobA, self.procJobB, self.procJobC, self.mergeJobA, self.mergeJobB, self.mergeJobC, self.cleanupJobA, self.cleanupJobB, self.cleanupJobC ] changeStateAction.execute(jobList) if baAPI: for job in jobList: job['plugin'] = 'TestPlugin' job['userdn'] = userDN job['usergroup'] = 'DEFAULT' job['userrole'] = 'DEFAULT' job['custom']['location'] = 'site1' baAPI.createNewJobs(wmbsJobs=jobList) # We'll create an unrelated workflow to verify that it isn't affected # by the killing code. bogusFileset = Fileset("dontkillme") bogusFileset.create() bogusFileA = File("bogus/lfnA", locations="goodse.cern.ch") bogusFileA.create() bogusFileset.addFile(bogusFileA) bogusFileset.commit() bogusWorkflow = Workflow(spec="spec2", owner="Steve", name="Bogus", task="Proc") bogusWorkflow.create() self.bogusSub = Subscription(fileset=bogusFileset, workflow=bogusWorkflow, type="Processing") self.bogusSub.create() self.bogusSub.acquireFiles(bogusFileA) return
def testTruncatedWFInsertion(self): """ _testTruncatedWFInsertion_ """ resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', seName='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertSite(siteName='site2', seName='goodse2.cern.ch', ceName='site2', plugin="TestPlugin") testWorkload = self.createTestWMSpec() testTopLevelTask = getFirstTask(testWorkload) testWMBSHelper = WMBSHelper(testWorkload, testTopLevelTask.name(), "SomeBlock", cachepath=self.workDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testTopLevelTask, testWMBSHelper.topLevelFileset) testWorkload.truncate("ResubmitTestWorkload", "/TestWorkload/ProcessingTask/MergeTask", "someserver", "somedatabase") # create the subscription for multiple top task (MergeTask and CleanupTask for the same block) for task in testWorkload.getTopLevelTask(): testResubmitWMBSHelper = WMBSHelper(testWorkload, task.name(), "SomeBlock2", cachepath=self.workDir) testResubmitWMBSHelper.createTopLevelFileset() testResubmitWMBSHelper._createSubscriptionsInWMBS( task, testResubmitWMBSHelper.topLevelFileset) mergeWorkflow = Workflow(name="ResubmitTestWorkload", task="/ResubmitTestWorkload/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0][ "output_fileset"] unmergedMergeOutput.loadData() self.assertEqual(unmergedMergeOutput.name, "/ResubmitTestWorkload/MergeTask/merged-Merged", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow( name="ResubmitTestWorkload", task="/ResubmitTestWorkload/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][ "merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][ "output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][ "merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][ "output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual( mergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual( unmergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Unmerged output fileset is wrong.") self.assertEqual( mergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset( name="ResubmitTestWorkload-MergeTask-SomeBlock2") topLevelFileset.loadData() mergeSubscription = Subscription(fileset=topLevelFileset, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset=unmergedMergeOutput, workflow=skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return
def testInsert(self): """ _testInsert_ Verify that inserting sites and thresholds works correctly, even if the site or threshold already exists. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1") myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1") myResourceControl.insertSite("testSite2", 100, 200, "testSE2", "testCE2") myResourceControl.insertThreshold("testSite1", "Processing", 20, 10) myResourceControl.insertThreshold("testSite1", "Merge", 200, 100) myResourceControl.insertThreshold("testSite1", "Merge", 250, 150) myResourceControl.insertThreshold("testSite2", "Processing", 50, 30) myResourceControl.insertThreshold("testSite2", "Merge", 135, 100) createThresholds = myResourceControl.listThresholdsForCreate() self.assertEqual(len(createThresholds.keys()), 2, "Error: Wrong number of site in Resource Control DB") self.assertTrue("testSite1" in createThresholds.keys(), "Error: Test Site 1 missing from thresholds.") self.assertTrue("testSite2" in createThresholds.keys(), "Error: Test Site 2 missing from thresholds.") self.assertEqual(createThresholds["testSite1"]["total_slots"], 10, "Error: Wrong number of total slots.") self.assertEqual( createThresholds["testSite1"]["pending_jobs"], 0, "Error: Wrong number of running jobs: %s" % createThresholds["testSite1"]["pending_jobs"]) self.assertEqual(createThresholds["testSite2"]["total_slots"], 100, "Error: Wrong number of total slots.") self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 0, "Error: Wrong number of running jobs.") thresholds = myResourceControl.listThresholdsForSubmit() self.assertEqual( len(thresholds.keys()), 2, "Error: Wrong number of sites in Resource Control DB") self.assertTrue("testSite1" in thresholds.keys(), "Error: testSite1 missing from thresholds.") self.assertTrue("testSite2" in thresholds.keys(), "Error: testSite2 missing from thresholds.") site1Info = thresholds["testSite1"] site2Info = thresholds["testSite2"] site1Thresholds = site1Info["thresholds"] site2Thresholds = site2Info["thresholds"] procThreshold1 = None procThreshold2 = None mergeThreshold1 = None mergeThreshold2 = None for threshold in site1Thresholds: if threshold["task_type"] == "Merge": mergeThreshold1 = threshold elif threshold["task_type"] == "Processing": procThreshold1 = threshold for threshold in site2Thresholds: if threshold["task_type"] == "Merge": mergeThreshold2 = threshold elif threshold["task_type"] == "Processing": procThreshold2 = threshold self.assertEqual(len(site1Thresholds), 2, "Error: Wrong number of task types.") self.assertEqual(len(site2Thresholds), 2, "Error: Wrong number of task types.") self.assertNotEqual(procThreshold1, None) self.assertNotEqual(procThreshold2, None) self.assertNotEqual(mergeThreshold1, None) self.assertNotEqual(mergeThreshold2, None) self.assertEqual(site1Info["total_pending_slots"], 10, "Error: Site thresholds wrong") self.assertEqual(site1Info["total_running_slots"], 20, "Error: Site thresholds wrong") self.assertEqual(site1Info["total_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(site1Info["total_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["max_slots"], 20, "Error: Site thresholds wrong") self.assertEqual(procThreshold1["pending_slots"], 10, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["max_slots"], 250, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold1["pending_slots"], 150, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_pending_slots"], 100, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_running_slots"], 200, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(site2Info["total_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["max_slots"], 50, "Error: Site thresholds wrong") self.assertEqual(procThreshold2["pending_slots"], 30, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["task_running_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["task_pending_jobs"], 0, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["max_slots"], 135, "Error: Site thresholds wrong") self.assertEqual(mergeThreshold2["pending_slots"], 100, "Error: Site thresholds wrong")
def setUp(self): """ setup for test. """ myThread = threading.currentThread() self.testInit = TestInit(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection() self.tearDown() self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMCore.BossAir", "WMCore.ResourceControl", "WMCore.Agent.Database" ], useDefault=False) self.testInit.setupCouch("bossair_t/jobs", "JobDump") self.testInit.setupCouch("bossair_t/fwjrs", "FWJRDump") self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs") #Create sites in resourceControl resourceControl = ResourceControl() for site in self.sites: resourceControl.insertSite(siteName=site, pnn='se.%s' % (site), cmsName=site, ceName=site, plugin="CondorPlugin", pendingSlots=1000, runningSlots=2000) resourceControl.insertThreshold(siteName = site, taskType = 'Processing', \ maxSlots = 1000, pendingSlots = 1000) resourceControl.insertSite(siteName='Xanadu', pnn='se.Xanadu', cmsName=site, ceName='Xanadu', plugin="TestPlugin") resourceControl.insertThreshold(siteName = 'Xanadu', taskType = 'Processing', \ maxSlots = 10000, pendingSlots = 10000) resourceControl.insertSite(siteName='jade-cms.hip.fi', pnn='madhatter.csc.fi', cmsName=site, ceName='jade-cms.hip.fi', plugin="ARCPlugin") resourceControl.insertThreshold(siteName = 'jade-cms.hip.fi', taskType = 'Processing', \ maxSlots = 100, pendingSlots = 100) # using this for glite submissions resourceControl.insertSite(siteName='grid-ce-01.ba.infn.it', pnn='storm-se-01.ba.infn.it', cmsName=site, ceName='grid-ce-01.ba.infn.it', plugin='gLitePlugin') resourceControl.insertThreshold(siteName = 'grid-ce-01.ba.infn.it', taskType = 'Processing', \ maxSlots = 50, pendingSlots = 50) # Create user newuser = self.daoFactory(classname="Users.New") newuser.execute(dn="tapas", group_name="phgroup", role_name="cmsrole") # We actually need the user name self.user = getpass.getuser() # Change this to the working dir to keep track of error and log files from condor self.testDir = self.testInit.generateWorkDir() # Set heartbeat componentName = 'test' self.heartbeatAPI = HeartbeatAPI(componentName) self.heartbeatAPI.registerComponent() componentName = 'JobTracker' self.heartbeatAPI2 = HeartbeatAPI(componentName) self.heartbeatAPI2.registerComponent() return
def testList(self): """ _testList_ Test the functions that list thresholds for creating jobs and submitting jobs. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "LsfPlugin") myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2", "T3_US_FNAL", "LsfPlugin") myResourceControl.insertThreshold("testSite1", "Processing", 20, 10) myResourceControl.insertThreshold("testSite1", "Merge", 200, 100) myResourceControl.insertThreshold("testSite2", "Processing", 50, 25) myResourceControl.insertThreshold("testSite2", "Merge", 135, 65) self.createJobs() createThresholds = myResourceControl.listThresholdsForCreate() submitThresholds = myResourceControl.listThresholdsForSubmit() self.assertEqual( len(createThresholds.keys()), 2, "Error: Wrong number of sites in create thresholds" ) self.assertEqual( createThresholds["testSite1"]["total_slots"], 10, "Error: Wrong number of slots for site 1" ) self.assertEqual( createThresholds["testSite2"]["total_slots"], 20, "Error: Wrong number of slots for site 2" ) # We should have two running jobs with locations at site one, # two running jobs without locations at site two, and one running # job without a location at site one and two. self.assertEqual( createThresholds["testSite1"]["pending_jobs"], {0 : 4}, "Error: Wrong number of pending jobs for site 1" ) # We should have one running job with a location at site 2 and # another running job without a location. self.assertEqual( createThresholds["testSite2"]["pending_jobs"], {0 : 2}, "Error: Wrong number of pending jobs for site 2" ) # We should also have a phedex_name self.assertEqual(createThresholds["testSite1"]["cms_name"], "T1_US_FNAL") self.assertEqual(createThresholds["testSite2"]["cms_name"], "T3_US_FNAL") mergeThreshold1 = None mergeThreshold2 = None procThreshold1 = None procThreshold2 = None self.assertEqual(set(submitThresholds.keys()), set(["testSite1", "testSite2"])) for taskType, threshold in submitThresholds["testSite1"]["thresholds"].items(): if taskType == "Merge": mergeThreshold1 = threshold elif taskType == "Processing": procThreshold1 = threshold for taskType, threshold in submitThresholds["testSite2"]["thresholds"].items(): if taskType == "Merge": mergeThreshold2 = threshold elif taskType == "Processing": procThreshold2 = threshold self.assertEqual(submitThresholds["testSite1"]["total_running_jobs"], 0, "Error: Wrong number of running jobs for submit thresholds.") self.assertEqual(submitThresholds["testSite2"]["total_running_jobs"], 1, "Error: Wrong number of running jobs for submit thresholds.") self.assertEqual(submitThresholds["testSite1"]["total_pending_jobs"], 1, "Error: Wrong number of pending jobs for submit thresholds.") self.assertEqual(submitThresholds["testSite2"]["total_pending_jobs"], 0, "Error: Wrong number of pending jobs for submit thresholds.") self.assertEqual(mergeThreshold1["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(mergeThreshold1["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold1["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold1["task_pending_jobs"], 1, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(mergeThreshold2["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(mergeThreshold2["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold2["task_running_jobs"], 1, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual(procThreshold2["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") return
def testList(self): """ _testList_ Test the functions that list thresholds for creating jobs and submitting jobs. """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "LsfPlugin") myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2") myResourceControl.insertThreshold("testSite1", "Processing", 20, 10) myResourceControl.insertThreshold("testSite1", "Merge", 200, 100) myResourceControl.insertThreshold("testSite2", "Processing", 50, 25) myResourceControl.insertThreshold("testSite2", "Merge", 135, 65) testWorkflow = Workflow(spec=makeUUID(), owner="Steve", name=makeUUID(), task="Test") testWorkflow.create() testFilesetA = Fileset(name="TestFilesetA") testFilesetA.create() testFilesetB = Fileset(name="TestFilesetB") testFilesetB.create() testFilesetC = Fileset(name="TestFilesetC") testFilesetC.create() testFileA = File(lfn="testFileA", locations=set(["testSE1", "testSE2"])) testFileA.create() testFilesetA.addFile(testFileA) testFilesetA.commit() testFilesetB.addFile(testFileA) testFilesetB.commit() testFilesetC.addFile(testFileA) testFilesetC.commit() testSubscriptionA = Subscription(fileset=testFilesetA, workflow=testWorkflow, type="Processing") testSubscriptionA.create() testSubscriptionA.addWhiteBlackList([{ "site_name": "testSite1", "valid": True }]) testSubscriptionB = Subscription(fileset=testFilesetB, workflow=testWorkflow, type="Processing") testSubscriptionB.create() testSubscriptionB.addWhiteBlackList([{ "site_name": "testSite1", "valid": False }]) testSubscriptionC = Subscription(fileset=testFilesetC, workflow=testWorkflow, type="Merge") testSubscriptionC.create() testJobGroupA = JobGroup(subscription=testSubscriptionA) testJobGroupA.create() testJobGroupB = JobGroup(subscription=testSubscriptionB) testJobGroupB.create() testJobGroupC = JobGroup(subscription=testSubscriptionC) testJobGroupC.create() # Site1, Has been assigned a location and is complete. testJobA = Job(name="testJobA", files=[testFileA]) testJobA["couch_record"] = makeUUID() testJobA.create(group=testJobGroupA) testJobA["state"] = "success" # Site 1, Has been assigned a location and is incomplete. testJobB = Job(name="testJobB", files=[testFileA]) testJobB["couch_record"] = makeUUID() testJobB.create(group=testJobGroupA) testJobB["state"] = "executing" runJobB = RunJob() runJobB.buildFromJob(testJobB) runJobB["status"] = "PEND" # Does not have a location, white listed to site 1 testJobC = Job(name="testJobC", files=[testFileA]) testJobC["couch_record"] = makeUUID() testJobC.create(group=testJobGroupA) testJobC["state"] = "new" # Site 2, Has been assigned a location and is complete. testJobD = Job(name="testJobD", files=[testFileA]) testJobD["couch_record"] = makeUUID() testJobD.create(group=testJobGroupB) testJobD["state"] = "success" # Site 2, Has been assigned a location and is incomplete. testJobE = Job(name="testJobE", files=[testFileA]) testJobE["couch_record"] = makeUUID() testJobE.create(group=testJobGroupB) testJobE["state"] = "executing" runJobE = RunJob() runJobE.buildFromJob(testJobE) runJobE["status"] = "RUN" # Does not have a location, site 1 is blacklisted. testJobF = Job(name="testJobF", files=[testFileA]) testJobF["couch_record"] = makeUUID() testJobF.create(group=testJobGroupB) testJobF["state"] = "new" # Site 3, Has been assigned a location and is complete. testJobG = Job(name="testJobG", files=[testFileA]) testJobG["couch_record"] = makeUUID() testJobG.create(group=testJobGroupC) testJobG["state"] = "cleanout" # Site 3, Has been assigned a location and is incomplete. testJobH = Job(name="testJobH", files=[testFileA]) testJobH["couch_record"] = makeUUID() testJobH.create(group=testJobGroupC) testJobH["state"] = "new" # Site 3, Does not have a location. testJobI = Job(name="testJobI", files=[testFileA]) testJobI["couch_record"] = makeUUID() testJobI.create(group=testJobGroupC) testJobI["state"] = "new" # Site 3, Does not have a location and is in cleanout. testJobJ = Job(name="testJobJ", files=[testFileA]) testJobJ["couch_record"] = makeUUID() testJobJ.create(group=testJobGroupC) testJobJ["state"] = "cleanout" changeStateAction = self.daoFactory(classname="Jobs.ChangeState") changeStateAction.execute(jobs=[ testJobA, testJobB, testJobC, testJobD, testJobE, testJobF, testJobG, testJobH, testJobI, testJobJ ]) self.insertRunJob.execute([runJobB, runJobE]) setLocationAction = self.daoFactory(classname="Jobs.SetLocation") setLocationAction.execute(testJobA["id"], "testSite1") setLocationAction.execute(testJobB["id"], "testSite1") setLocationAction.execute(testJobD["id"], "testSite1") setLocationAction.execute(testJobE["id"], "testSite1") setLocationAction.execute(testJobG["id"], "testSite1") setLocationAction.execute(testJobH["id"], "testSite1") createThresholds = myResourceControl.listThresholdsForCreate() submitThresholds = myResourceControl.listThresholdsForSubmit() self.assertEqual(len(createThresholds.keys()), 2, "Error: Wrong number of sites in create thresholds") self.assertEqual(createThresholds["testSite1"]["total_slots"], 10, "Error: Wrong number of slots for site 1") self.assertEqual(createThresholds["testSite2"]["total_slots"], 20, "Error: Wrong number of slots for site 2") # We should have two running jobs with locations at site one, # two running jobs without locations at site two, and one running # job without a location at site one and two. self.assertEqual(createThresholds["testSite1"]["pending_jobs"], 4, "Error: Wrong number of pending jobs for site 1") # We should have one running job with a location at site 2 and # another running job without a location. self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 2, "Error: Wrong number of pending jobs for site 2") # We should also have a phedex_name self.assertEqual(createThresholds["testSite1"]["cms_name"], "T1_US_FNAL") self.assertEqual(createThresholds["testSite2"]["cms_name"], None) mergeThreshold1 = None mergeThreshold2 = None procThreshold1 = None procThreshold2 = None self.assertEqual(submitThresholds["testSite1"]['cms_name'], 'T1_US_FNAL') for threshold in submitThresholds["testSite1"]["thresholds"]: if threshold['task_type'] == "Merge": mergeThreshold1 = threshold elif threshold['task_type'] == "Processing": procThreshold1 = threshold self.assertEqual(submitThresholds["testSite2"]['cms_name'], None) for threshold in submitThresholds["testSite2"]["thresholds"]: if threshold['task_type'] == "Merge": mergeThreshold2 = threshold elif threshold['task_type'] == "Processing": procThreshold2 = threshold self.assertEqual( submitThresholds["testSite1"]["total_running_jobs"], 1, "Error: Wrong number of running jobs for submit thresholds.") self.assertEqual( submitThresholds["testSite2"]["total_running_jobs"], 0, "Error: Wrong number of running jobs for submit thresholds.") self.assertEqual( submitThresholds["testSite1"]["total_pending_jobs"], 1, "Error: Wrong number of pending jobs for submit thresholds.") self.assertEqual( submitThresholds["testSite2"]["total_pending_jobs"], 0, "Error: Wrong number of pending jobs for submit thresholds.") self.assertEqual( mergeThreshold1["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( mergeThreshold1["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( procThreshold1["task_running_jobs"], 1, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( procThreshold1["task_pending_jobs"], 1, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( mergeThreshold2["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( mergeThreshold2["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( procThreshold2["task_running_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") self.assertEqual( procThreshold2["task_pending_jobs"], 0, "Error: Wrong number of task running jobs for submit thresholds.") return
def setupForKillTest(self, baAPI = None): """ _setupForKillTest_ Inject a workflow into WMBS that has a processing task, a merge task and a cleanup task. Inject files into the various tasks at various processing states (acquired, complete, available...). Also create jobs for each subscription in various states. """ myThread = threading.currentThread() daoFactory = DAOFactory(package = "WMCore.WMBS", logger = myThread.logger, dbinterface = myThread.dbi) locationAction = daoFactory(classname = "Locations.New") changeStateAction = daoFactory(classname = "Jobs.ChangeState") resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'site1', seName = 'goodse.cern.ch', ceName = 'site1', plugin = "TestPlugin") resourceControl.insertThreshold(siteName = 'site1', taskType = 'Processing', \ maxSlots = 10000) inputFileset = Fileset("input") inputFileset.create() inputFileA = File("lfnA", locations = "goodse.cern.ch") inputFileB = File("lfnB", locations = "goodse.cern.ch") inputFileC = File("lfnC", locations = "goodse.cern.ch") inputFileA.create() inputFileB.create() inputFileC.create() inputFileset.addFile(inputFileA) inputFileset.addFile(inputFileB) inputFileset.addFile(inputFileC) inputFileset.commit() unmergedOutputFileset = Fileset("unmerged") unmergedOutputFileset.create() unmergedFileA = File("ulfnA", locations = "goodse.cern.ch") unmergedFileB = File("ulfnB", locations = "goodse.cern.ch") unmergedFileC = File("ulfnC", locations = "goodse.cern.ch") unmergedFileA.create() unmergedFileB.create() unmergedFileC.create() unmergedOutputFileset.addFile(unmergedFileA) unmergedOutputFileset.addFile(unmergedFileB) unmergedOutputFileset.addFile(unmergedFileC) unmergedOutputFileset.commit() mainProcWorkflow = Workflow(spec = "spec1", owner = "Steve", name = "Main", task = "Proc") mainProcWorkflow.create() mainProcMergeWorkflow = Workflow(spec = "spec1", owner = "Steve", name = "Main", task = "ProcMerge") mainProcMergeWorkflow.create() mainCleanupWorkflow = Workflow(spec = "spec1", owner = "Steve", name = "Main", task = "Cleanup") mainCleanupWorkflow.create() self.mainProcSub = Subscription(fileset = inputFileset, workflow = mainProcWorkflow, type = "Processing") self.mainProcSub.create() self.mainProcSub.acquireFiles(inputFileA) self.mainProcSub.completeFiles(inputFileB) procJobGroup = JobGroup(subscription = self.mainProcSub) procJobGroup.create() self.procJobA = Job(name = "ProcJobA") self.procJobA["state"] = "new" self.procJobA["location"] = "site1" self.procJobB = Job(name = "ProcJobB") self.procJobB["state"] = "executing" self.procJobB["location"] = "site1" self.procJobC = Job(name = "ProcJobC") self.procJobC["state"] = "complete" self.procJobC["location"] = "site1" self.procJobA.create(procJobGroup) self.procJobB.create(procJobGroup) self.procJobC.create(procJobGroup) self.mainMergeSub = Subscription(fileset = unmergedOutputFileset, workflow = mainProcMergeWorkflow, type = "Merge") self.mainMergeSub.create() self.mainMergeSub.acquireFiles(unmergedFileA) self.mainMergeSub.failFiles(unmergedFileB) mergeJobGroup = JobGroup(subscription = self.mainMergeSub) mergeJobGroup.create() self.mergeJobA = Job(name = "MergeJobA") self.mergeJobA["state"] = "exhausted" self.mergeJobA["location"] = "site1" self.mergeJobB = Job(name = "MergeJobB") self.mergeJobB["state"] = "cleanout" self.mergeJobB["location"] = "site1" self.mergeJobC = Job(name = "MergeJobC") self.mergeJobC["state"] = "new" self.mergeJobC["location"] = "site1" self.mergeJobA.create(mergeJobGroup) self.mergeJobB.create(mergeJobGroup) self.mergeJobC.create(mergeJobGroup) self.mainCleanupSub = Subscription(fileset = unmergedOutputFileset, workflow = mainCleanupWorkflow, type = "Cleanup") self.mainCleanupSub.create() self.mainCleanupSub.acquireFiles(unmergedFileA) self.mainCleanupSub.completeFiles(unmergedFileB) cleanupJobGroup = JobGroup(subscription = self.mainCleanupSub) cleanupJobGroup.create() self.cleanupJobA = Job(name = "CleanupJobA") self.cleanupJobA["state"] = "new" self.cleanupJobA["location"] = "site1" self.cleanupJobB = Job(name = "CleanupJobB") self.cleanupJobB["state"] = "executing" self.cleanupJobB["location"] = "site1" self.cleanupJobC = Job(name = "CleanupJobC") self.cleanupJobC["state"] = "complete" self.cleanupJobC["location"] = "site1" self.cleanupJobA.create(cleanupJobGroup) self.cleanupJobB.create(cleanupJobGroup) self.cleanupJobC.create(cleanupJobGroup) jobList = [self.procJobA, self.procJobB, self.procJobC, self.mergeJobA, self.mergeJobB, self.mergeJobC, self.cleanupJobA, self.cleanupJobB, self.cleanupJobC] changeStateAction.execute(jobList) if baAPI: for job in jobList: job['plugin'] = 'TestPlugin' job['userdn'] = 'Steve' job['custom']['location'] = 'site1' baAPI.createNewJobs(wmbsJobs = jobList) # We'll create an unrelated workflow to verify that it isn't affected # by the killing code. bogusFileset = Fileset("dontkillme") bogusFileset.create() bogusFileA = File("bogus/lfnA", locations = "goodse.cern.ch") bogusFileA.create() bogusFileset.addFile(bogusFileA) bogusFileset.commit() bogusWorkflow = Workflow(spec = "spec2", owner = "Steve", name = "Bogus", task = "Proc") bogusWorkflow.create() self.bogusSub = Subscription(fileset = bogusFileset, workflow = bogusWorkflow, type = "Processing") self.bogusSub.create() self.bogusSub.acquireFiles(bogusFileA) return
def testTruncatedWFInsertion(self): """ _testTruncatedWFInsertion_ """ resourceControl = ResourceControl() resourceControl.insertSite(siteName = 'site1', seName = 'goodse.cern.ch', ceName = 'site1', plugin = "TestPlugin") resourceControl.insertSite(siteName = 'site2', seName = 'goodse2.cern.ch', ceName = 'site2', plugin = "TestPlugin") testWorkload = self.createTestWMSpec() testWMBSHelper = WMBSHelper(testWorkload, "SomeBlock", cachepath = self.workDir) testWMBSHelper.createSubscription() testWorkload.truncate("ResubmitTestWorkload", "/TestWorkload/ProcessingTask/MergeTask", "someserver", "somedatabase") testResubmitWMBSHelper = WMBSHelper(testWorkload, "SomeBlock2", cachepath = self.workDir) testResubmitWMBSHelper.createSubscription() mergeWorkflow = Workflow(name = "ResubmitTestWorkload", task = "/ResubmitTestWorkload/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow(name = "ResubmitTestWorkload", task = "/ResubmitTestWorkload/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap.keys()), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0]["output_fileset"] unmergedMergeOutput.loadData() self.assertEqual(unmergedMergeOutput.name, "/ResubmitTestWorkload/MergeTask/merged-Merged", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow(name = "ResubmitTestWorkload", task = "/ResubmitTestWorkload/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual(skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual(mergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual(unmergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Unmerged output fileset is wrong.") self.assertEqual(mergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name = "ResubmitTestWorkload-MergeTask-SomeBlock2") topLevelFileset.loadData() mergeSubscription = Subscription(fileset = topLevelFileset, workflow = mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset = unmergedMergeOutput, workflow = skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return
def testThresholdPriority(self): """ _testThresholdPriority_ Test that we get things back in priority order """ myResourceControl = ResourceControl() myResourceControl.insertSite("testSite1", 20, 40, "testSE1", "testCE1") myResourceControl.insertThreshold("testSite1", "Processing", 10, 8, priority=1) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3, priority=2) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Processing') myResourceControl.insertThreshold("testSite1", "Processing", 10, 8, priority=2) myResourceControl.insertThreshold("testSite1", "Merge", 5, 3, priority=1) # Should now be in reverse order result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Processing') myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2") myResourceControl.insertThreshold("testSite2", "Processing", 10, 8, priority=1) myResourceControl.insertThreshold("testSite2", "Merge", 5, 3, priority=2) # Should be in proper order for site 2 result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite2']['thresholds'][0]['task_type'], 'Merge') self.assertEqual(result['testSite2']['thresholds'][1]['task_type'], 'Processing') # Should now be in reverse order for site 1 self.assertEqual(result['testSite1']['thresholds'][1]['task_type'], 'Merge') self.assertEqual(result['testSite1']['thresholds'][0]['task_type'], 'Processing') myResourceControl.insertThreshold("testSite2", "Merge", 20, 10) result = myResourceControl.listThresholdsForSubmit() self.assertEqual(result['testSite2']['thresholds'][0]['priority'], 2) return
def testCreateSubscription(self): """ _testCreateSubscription_ Verify that the subscription creation code works correctly. """ resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', seName='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertSite(siteName='site2', seName='goodse2.cern.ch', ceName='site2', plugin="TestPlugin") testWorkload = self.createTestWMSpec() testTopLevelTask = getFirstTask(testWorkload) testWMBSHelper = WMBSHelper(testWorkload, testTopLevelTask.name(), "SomeBlock", cachepath=self.workDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testTopLevelTask, testWMBSHelper.topLevelFileset) procWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/ProcessingTask") procWorkflow.load() self.assertEqual(procWorkflow.owner, "sfoulkes", "Error: Wrong owner: %s" % procWorkflow.owner) self.assertEqual(procWorkflow.group, "DMWM", "Error: Wrong group: %s" % procWorkflow.group) self.assertEqual(procWorkflow.wfType, "TestReReco", "Error: Wrong type.") self.assertEqual( procWorkflow.spec, os.path.join(self.workDir, procWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(procWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") mergedProcOutput = procWorkflow.outputMap["OutputA"][0][ "merged_output_fileset"] unmergedProcOutput = procWorkflow.outputMap["OutputA"][0][ "output_fileset"] mergedProcOutput.loadData() unmergedProcOutput.loadData() self.assertEqual( mergedProcOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-Merged", "Error: Merged output fileset is wrong.") self.assertEqual(unmergedProcOutput.name, "/TestWorkload/ProcessingTask/unmerged-OutputA", "Error: Unmerged output fileset is wrong.") mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/ProcessingTask/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap.keys()), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/ProcessingTask/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap.keys()), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["Merged"][0][ "output_fileset"] unmergedMergeOutput.loadData() self.assertEqual( unmergedMergeOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-Merged", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/ProcessingTask/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap.keys()), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][ "merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputA"][0][ "output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][ "merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputB"][0][ "output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual( mergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual( unmergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputA", "Error: Unmerged output fileset is wrong.") self.assertEqual( mergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-ProcessingTask-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset=topLevelFileset, workflow=procWorkflow) procSubscription.loadData() self.assertEqual(len(procSubscription.getWhiteBlackList()), 2, "Error: Wrong site white/black list for proc sub.") for site in procSubscription.getWhiteBlackList(): if site["site_name"] == "site1": self.assertEqual(site["valid"], 1, "Error: Site should be white listed.") else: self.assertEqual(site["valid"], 0, "Error: Site should be black listed.") self.assertEqual(procSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") mergeSubscription = Subscription(fileset=unmergedProcOutput, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset=unmergedMergeOutput, workflow=skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return