def testB_TestBase(self): """ _TestBase_ See if we can use the defaults or an environment variable to build a testBase NOTE: This has to run in WMCore/test/python/WMCore_t, and so is listed as integration. """ test = os.path.normpath( os.path.join(os.getcwd(), '../../../test/python')) base = getTestBase() self.assertEqual(base, test) base = getTestBase(importFlag=False) self.assertEqual(base, test) os.environ['WMCORE_TEST_ROOT'] = '/thisdirectoryshouldneverexist' base = getTestBase(importFlag=False) self.assertEqual(base, test) os.environ['WMCORE_TEST_ROOT'] = '/tmp' base = getTestBase(importFlag=False) self.assertEqual(base, '/tmp') return
def testGetIterMatchObjectOnRegex(self): logPath = os.path.join(getTestBase(), "WMCore_t/test_empty.log") for mo in getIterMatchObjectOnRegexp(logPath, WMEXCEPTION_REGEXP): pass count = 0 ecount = 0 logPath = os.path.join(getTestBase(), "WMCore_t/test_condor.log") for mo in getIterMatchObjectOnRegexp(logPath, WMEXCEPTION_REGEXP): errMsg = mo.group("WMException") if errMsg: count += 1 error = mo.group("ERROR") if error: ecount += 1 self.assertEqual(count, 4) self.assertEqual(ecount, 1) rcount = 0 scount = 0 for mo in getIterMatchObjectOnRegexp(logPath, CONDOR_LOG_FILTER_REGEXP): if mo.group("Reason"): reason = mo.group("Reason") rcount += 1 if mo.group("Site"): site = mo.group("Site") scount += 1 self.assertEqual(rcount, 1) self.assertEqual(scount, 2) self.assertEqual(site, 'T1_US_FNAL') self.assertEqual(reason, 'via condor_rm (by user cmst1)')
def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] #Here Test the CondorPlugin instead of MockPlugin #config.BossAir.pluginNames = ['CondorPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join( self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
def getConfig(self): """ _getConfig_ Gets a basic config from default location """ config = Configuration() config.component_("Agent") config.Agent.WMSpecDirectory = self.testDir config.Agent.agentName = 'testAgent' config.Agent.componentName = self.componentName config.Agent.useHeartbeat = False #First the general stuff config.section_("General") config.General.workDir = os.getenv("TESTDIR", self.testDir) #Now the CoreDatabase information config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") # BossAir and MockPlugin configuration config.section_("BossAir") config.BossAir.pluginNames = ['MockPlugin'] config.BossAir.pluginDir = 'WMCore.BossAir.Plugins' config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction'] config.BossAir.nCondorProcesses = 1 config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh") # JobSubmitter configuration config.component_("JobSubmitter") config.JobSubmitter.logLevel = 'DEBUG' config.JobSubmitter.maxThreads = 1 config.JobSubmitter.pollInterval = 10 config.JobSubmitter.submitScript = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components') config.JobSubmitter.workerThreads = 2 config.JobSubmitter.jobsPerWorker = 200 #JobStateMachine config.component_('JobStateMachine') config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "jobsubmitter_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' # Needed, because this is a test os.makedirs(config.JobSubmitter.componentDir) return config
def testChecksum(self): """ Tests checksum method """ checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz')) checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz')) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'}) return
def testChecksum(self): """ Tests checksum method """ self.ufc = UserFileCache() checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz')) checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz')) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'}) return
def stuffWMBS(self): """ _stuffWMBS_ Inject the workflow in WMBS and add the subscriptions """ testWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "") testWorkflow.create() testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(), "WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"), owner = "/CN=OU/DN=SomeoneWithPermissions", name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "") testMergeWorkflow.create() testWMBSFileset = Fileset(name = "TopFileset") testWMBSFileset.create() testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset") testWMBSFilesetUnmerged.create() testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10) testFileA.addRun(Run(10, *[12312])) testFileA.setLocation('malpaquet') testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10) testFileB.addRun(Run(10, *[12314])) testFileB.setLocation('malpaquet') testFileA.create() testFileB.create() testWMBSFileset.addFile(testFileA) testWMBSFilesetUnmerged.addFile(testFileB) testWMBSFileset.commit() testWMBSFilesetUnmerged.commit() testSubscription = Subscription(fileset = testWMBSFileset, workflow = testWorkflow) testSubscription.create() testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged, workflow = testMergeWorkflow, type = "Merge") testSubscriptionMerge.create() return (testSubscription, testSubscriptionMerge)
def _configCacheId(self, label): """Return config cache id for given config label""" key, cert = self.__class__.reqmgr['requests'].getKeyCert() configCache = ConfigCache(self.__class__.endpoint + '/couchdb', 'reqmgr_config_cache', ckey=key, cert=cert) try: configCacheId = configCache.getIDFromLabel(label) except: configCacheId = None if configCacheId: return configCacheId # The following will fail if FWCore.ParameterSet not in PYTHONPATH from PSetTweaks.WMTweak import makeTweak configCache.createUserGroup('test', 'test') configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data', 'configs') configCache.addConfig(os.path.join(configDir, label + '.py')) configCache.setLabel(label) configCache.setDescription(label) modPath = imp.find_module(label, [configDir]) loadedConfig = imp.load_module(label, modPath[0], modPath[1], modPath[2]) configCache.setPSetTweaks( makeTweak(loadedConfig.process).jsondictionary()) configCache.save() return configCache.getIDFromLabel(label)
def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test', id = configCache.getCouchID(), rev = configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def testC_testViews(self): """ _testViews_ Prototype test for what should be a lot of other tests. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache.createUserGroup(groupname = "testGroup", username = '******') configCache.setPSetTweaks(PSetTweak = PSetTweak) configCache.attachments['attach1'] = attach configCache.document['md5_hash'] = "somemd5" psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig = psetPath, psetHash = None) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test') configCache2.document['md5_hash'] = configCache.document['md5_hash'] configCache2.load() self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache2.delete() return
def testTrivialFileCatalog(self): """ Run some simple tests on reading a trivialFileCatalog """ tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T1_US_FNAL_TrivialFileCatalog.xml") if not os.path.exists(tfcFilename): raise Exception("No TrivialFileCatalog found!") tfcInstance = readTFC(tfcFilename) self.assertEqual(type(tfcInstance), type(TrivialFileCatalog())) # Look for similarities in each node of the TFC file for mapping in ['lfn-to-pfn', 'pfn-to-lfn']: for x in tfcInstance[mapping]: self.assertEqual('path-match-expr' in x, True) self.assertEqual('path-match' in x, True) self.assertEqual('protocol' in x, True) self.assertEqual('result' in x, True) self.assertEqual('chain' in x, True) self.assertEqual(x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True, 'Could not find protocol %s' % (x['protocol'])) self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def setUp(self): """ _setUp_ Setup a dashboard reporter """ self.reporter = DashboardReporter(config=None) self.processingReport = ProcessingSample.report self.mergeReport = MergeSample.report self.errorReport = ErrorSample.report self.fallbackReport = FallbackSample.report self.twoFileFallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml") self.pileupXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWPileup.xml") return
def testBasic(self): """ _testBasic_ Test the basic functions of the DBSBufferDataset, create, load, exists and also the ability to add subscriptions. """ originalDataset = DBSBufferDataset(path = '/bogus/bogus/go') originalDataset.create() myThread = threading.currentThread() result = myThread.dbi.processData("SELECT id FROM dbsbuffer_dataset")[0].fetchall() self.assertEqual(originalDataset.exists(), result[0][0]) duplicateDataset = DBSBufferDataset(path = '/bogus/bogus/go') duplicateDataset.create() self.assertEqual(originalDataset.exists(), duplicateDataset.exists()) result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset")[0].fetchall() self.assertEqual(result[0][0], 1) loadedDataset = DBSBufferDataset(path = '/bogus/bogus/go') loadedDataset.load() self.assertEqual(loadedDataset.exists(), originalDataset.exists()) secondDataset = DBSBufferDataset(path = '/BogusPrimary/Run2012Z-PromptReco-v1/RECO') secondDataset.create() workload = WMWorkloadHelper() workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl')) secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO']) secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO']) self.assertEqual(len(secondDataset['subscriptions']), 3) result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset_subscription")[0].fetchall() self.assertEqual(result[0][0], 3) return
def testGetIterMatchObjectOnRegex(self): count = 0 ecount = 0 logPath = os.path.join(getTestBase(), "WMCore_t/test_condor.log") for mo in getIterMatchObjectOnRegexp(logPath, WMEXCEPTION_REGEXP): errMsg = mo.group("WMException") if errMsg: count += 1 error = mo.group("ERROR") if error: ecount += 1 self.assertEqual(count, 4) self.assertEqual(ecount, 1) rcount = 0 scount = 0 for mo in getIterMatchObjectOnRegexp(logPath, CONDOR_LOG_FILTER_REGEXP): if mo.group("Reason"): reason = mo.group("Reason") rcount += 1 if mo.group("Site"): site = mo.group("Site") scount += 1 self.assertEqual(rcount, 1) self.assertEqual(scount, 2) self.assertEqual(site, 'T1_US_FNAL') self.assertEqual(reason, 'via condor_rm (by user cmst1)')
def createConfig(self): """ _createConfig_ Create a config for the JobSubmitter. These parameters are still pulled from the environment. """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = "jobsubmittercaching_t" config.section_("BossAir") config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.BossAir.pluginNames = ["SimpleCondorPlugin"] config.BossAir.nCondorProcesses = 1 config.component_("JobSubmitter") config.JobSubmitter.submitDir = self.testDir config.JobSubmitter.submitScript = os.path.join(getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') return config
def testPerformanceJSON(self): """ _testPerformanceJSON_ Verify that the performance section of the report is correctly converted to JSON. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) perfSection = myReport.__to_json__(thunker = None)["steps"]["cmsRun1"]["performance"] self.assertTrue(perfSection.has_key("storage"), "Error: Storage section is missing.") self.assertTrue(perfSection.has_key("memory"), "Error: Memory section is missing.") self.assertTrue(perfSection.has_key("cpu"), "Error: CPU section is missing.") self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105", "Error: AvgEventCPU is wrong.") self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703", "Error: TotalJobTime is wrong.") self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166, "Error: readTotalMB is wrong.") self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653, "Error: readMaxMSec is wrong") self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293", "Error: PeakValueRss is wrong.") self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281", "Error: PeakValueVsize is wrong.") return
def testreportFailureToDashboard(self): """ _testreportFailureToDashboard_ test report Failure to dashboard with/without job_ad file """ self.assertEqual(0, reportFailureToDashboard(0, None)) # parseAd is called by reportFailureToDashboard, but if we would # call it from reportFailureToDashboard error will be suppressed # and it will return same exit code # So lets call it directly if '_CONDOR_JOB_AD' in os.environ: del os.environ["_CONDOR_JOB_AD"] self.assertRaises(KeyError, parseAd) self.assertEqual(0, reportFailureToDashboard(0)) os.environ["_CONDOR_JOB_AD"] = "testFile" # if this environment variable is set, it will check this file # if file is not present, it will raise IOError self.assertRaises(IOError, parseAd) # Test real job_ad file and check if it returns dict. os.environ["_CONDOR_JOB_AD"] = os.path.join(getTestBase(), '..', 'data', 'WMCore', 'Services', 'Dashboard', 'job_ad_file') out = parseAd() self.assertTrue(isinstance(out, dict)) # Also test it sending directly parsed job ad self.assertEqual(0, reportFailureToDashboard(0, out))
def testB_ExecuteNonZeroExit(self): """ _ExecuteNonZeroExit_ Test the execution of a script which exits with non-zero code. """ self.step.application.command.executable = "brokenCmsRun.py" shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) try: os.chdir(self.step.builder.workingDir) executor = StepFactory.getStepExecutor("CMSSW") executor.initialise(self.step, self.job) executor.pre() executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset") try: executor.execute() self.fail("An exception should have been raised") except WMExecutionFailure, ex: executor.diagnostic(ex.code, executor, ExceptionInstance = ex) self.assertEqual(8001, executor.report.getExitCode()) report = Report() report.load("Report.pkl") self.assertEqual(8001, report.getExitCode()) except Exception, ex: self.fail("Failure encountered, %s" % str(ex))
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus", processedDataset = "Test-Era-v1", dataTier = "DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def testInsertT0(self): """ _testInsertT0_ Test to see if we can insert the Tier-0 alone with a single option """ self.createConfig() resControlPath = os.path.join(getTestBase(), "../../bin/wmagent-resource-control") env = os.environ env['PYTHONPATH'] = ":".join(sys.path) cmdline = [resControlPath, "--add-T0" ] retval = subprocess.Popen( cmdline, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, env = env) (_, _) = retval.communicate() myResourceControl = ResourceControl() result = myResourceControl.listThresholdsForSubmit() self.assertEqual(len(result), 1) self.assertTrue('CERN' in result) for x in result: self.assertEqual(len(result[x]['thresholds']), 9) self.assertEqual(result[x]['total_pending_slots'], 500) self.assertEqual(result[x]['total_running_slots'], -1) for taskType, thresh in result[x]['thresholds'].items(): if taskType == 'Processing': self.assertEqual(thresh['priority'], 0) self.assertEqual(thresh['max_slots'], -1) # Verify that sites with more than one SE were added correctly. cernInfo = myResourceControl.listSiteInfo("CERN") self.assertTrue(len(cernInfo["pnn"]) == 2) return
def createConfig(self): """ _createConfig_ Create a config and save it to the temp dir. Set the WMAGENT_CONFIG environment variable so the config gets picked up. """ config = Configuration() config.section_("General") config.General.workDir = os.getenv("TESTDIR", os.getcwd()) config.section_("Agent") config.Agent.componentName = "resource_control_t" config.section_("CoreDatabase") config.CoreDatabase.connectUrl = os.getenv("DATABASE") config.CoreDatabase.socket = os.getenv("DBSOCK") config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv('COUCHURL') config.JobStateMachine.couchDBName = "bossair_t" config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t' config.JobStateMachine.summaryStatsDBName = 'stat_summary_t' config.section_("BossAir") config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.BossAir.pluginNames = ["MockPlugin"] config.BossAir.section_("MockPlugin") config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(), 'WMComponent_t/JobAccountant_t/fwjrs', "MergeSuccess.pkl") configHandle = open(os.path.join(self.tempDir, "config.py"), "w") configHandle.write(str(config)) configHandle.close() os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py") return config
def testD_ExecuteNoOutput(self): """ _ExecuteNoOutput_ Test what happens when no output is produced, the proper error should be included. """ self.step.application.command.executable = "cmsRun.py" shutil.copy( os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) try: os.chdir(self.step.builder.workingDir) executor = StepFactory.getStepExecutor("CMSSW") executor.initialise(self.step, self.job) executor.pre() executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset") executor.execute() executor.post() self.assertEqual(60450, executor.report.getExitCode()) except Exception as ex: self.fail("Failure encountered, %s" % str(ex)) finally: os.chdir(self.oldCwd) return
def testInsertSite(self): """ _testInsertSite_ Test to see if we can insert a fake test site alone with a single option """ self.createConfig() resControlPath = os.path.join(getTestBase(), "../../bin/wmagent-resource-control") env = os.environ env['PYTHONPATH'] = ":".join(sys.path) cmdline = [resControlPath, "--add-Test"] retval = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env) (_, _) = retval.communicate() myResourceControl = ResourceControl() result = myResourceControl.listThresholdsForSubmit() self.assertEqual(len(result), 1) self.assertTrue('CERN' in result) for x in result: self.assertEqual(len(result[x]['thresholds']), 7) self.assertEqual(result[x]['total_pending_slots'], 500) self.assertEqual(result[x]['total_running_slots'], 1) for taskType, thresh in result[x]['thresholds'].items(): if taskType == 'Processing': self.assertEqual(thresh['priority'], 0) self.assertEqual(thresh['max_slots'], 1) # Verify that sites with more than one SE were added correctly. cernInfo = myResourceControl.listSiteInfo("CERN") self.assertTrue(len(cernInfo["pnn"]) == 2) return
def testJSONEncoding(self): """ _testJSONEncoding_ Verify that turning the FWJR into a JSON object works correctly. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) jsonReport = myReport.__to_json__(None) assert "task" in jsonReport.keys(), \ "Error: Task name missing from report." assert len(jsonReport["steps"].keys()) == 1, \ "Error: Wrong number of steps in report." assert "cmsRun1" in jsonReport["steps"].keys(), \ "Error: Step missing from json report." cmsRunStep = jsonReport["steps"]["cmsRun1"] jsonReportSections = ["status", "errors", "logs", "parameters", "site", "analysis", "cleanup", "input", "output", "start"] for jsonReportSection in jsonReportSections: assert jsonReportSection in cmsRunStep.keys(), \ "Error: missing section: %s" % jsonReportSection return
def testD_ExecuteNoOutput(self): """ _ExecuteNoOutput_ Test what happens when no output is produced, the proper error should be included. """ self.step.application.command.executable = "cmsRun.py" shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) try: os.chdir(self.step.builder.workingDir) executor = StepFactory.getStepExecutor("CMSSW") executor.initialise(self.step, self.job) executor.pre() executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset") executor.execute() executor.post() self.assertEqual(60450, executor.report.getExitCode()) except Exception as ex: self.fail("Failure encountered, %s" % str(ex)) finally: os.chdir(self.oldCwd) return
def testPerformanceJSON(self): """ _testPerformanceJSON_ Verify that the performance section of the report is correctly converted to JSON. """ xmlPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) perfSection = myReport.__to_json__( thunker=None)["steps"]["cmsRun1"]["performance"] self.assertTrue("storage" in perfSection, "Error: Storage section is missing.") self.assertTrue("memory" in perfSection, "Error: Memory section is missing.") self.assertTrue("cpu" in perfSection, "Error: CPU section is missing.") self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105", "Error: AvgEventCPU is wrong.") self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703", "Error: TotalJobTime is wrong.") self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166, "Error: readTotalMB is wrong.") self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653, "Error: readMaxMSec is wrong") self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293", "Error: PeakValueRss is wrong.") self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281", "Error: PeakValueVsize is wrong.") return
def testJSONEncoding(self): """ _testJSONEncoding_ Verify that turning the FWJR into a JSON object works correctly. """ xmlPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) jsonReport = myReport.__to_json__(None) assert "task" in jsonReport.keys(), \ "Error: Task name missing from report." assert len(jsonReport["steps"].keys()) == 1, \ "Error: Wrong number of steps in report." assert "cmsRun1" in jsonReport["steps"].keys(), \ "Error: Step missing from json report." cmsRunStep = jsonReport["steps"]["cmsRun1"] jsonReportSections = [ "status", "errors", "logs", "parameters", "site", "analysis", "cleanup", "input", "output", "start" ] for jsonReportSection in jsonReportSections: assert jsonReportSection in cmsRunStep.keys(), \ "Error: missing section: %s" % jsonReportSection return
def testPerformanceSummary(self): """ _testPerformanceSummary_ Test whether or not we can pull performance information out of a Timing/SimpleMemoryCheck jobReport """ xmlPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) # Do a brief check of the three sections perf = myReport.data.cmsRun1.performance self.assertEqual(perf.memory.PeakValueRss, '492.293') self.assertEqual(perf.cpu.TotalJobCPU, '9.16361') self.assertEqual(perf.storage.writeTotalMB, 5.22226) self.assertAlmostEqual(perf.storage.writeTotalSecs, 0, places=0) # actual value is 0.06 self.assertEqual(perf.storage.readPercentageOps, 0.98585512216030857) return
def publishPerformanceDashBoard(self, dashBoardUrl, PD, release, worthPoints): dashboardPayload = [] for instLuminosity in worthPoints: timePerEvent = int(worthPoints[instLuminosity]) dashboardPayload.append( { "primaryDataset": PD, "release": release, "integratedLuminosity": instLuminosity, "timePerEvent": timePerEvent, } ) data = '{"data":%s}' % str(dashboardPayload).replace("'", '"') # let's suppose it works.. testDashBoardPayloadFile = open( os.path.join(getTestBase(), "WMComponent_t/TaskArchiver_t/DashBoardPayload.json"), "r" ) testDashBoardPayload = testDashBoardPayloadFile.read() testDashBoardPayloadFile.close() self.assertEquals(data, testDashBoardPayload) return True
def setUp(self): """ _setUp_ Figure out the location of the XML report produced by CMSSW. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("report_t/fwjrs", "FWJRDump") testData = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t") self.xmlPath = os.path.join(testData, "CMSSWProcessingReport.xml") self.badxmlPath = os.path.join(testData, "CMSSWFailReport2.xml") self.skippedFilesxmlPath = os.path.join( testData, "CMSSWSkippedNonExistentFile.xml") self.skippedAllFilesxmlPath = os.path.join(testData, "CMSSWSkippedAll.xml") self.fallbackXmlPath = os.path.join(testData, "CMSSWInputFallback.xml") self.twoFileFallbackXmlPath = os.path.join(testData, "CMSSWTwoFileRemote.xml") self.pileupXmlPath = os.path.join(testData, "CMSSWPileup.xml") self.withEventsXmlPath = os.path.join(testData, "CMSSWWithEventCounts.xml") self.testDir = self.testInit.generateWorkDir() return
def publishPerformanceDashBoard(self, dashBoardUrl, PD, release, worthPoints): dashboardPayload = [] for instLuminosity in worthPoints: timePerEvent = int(worthPoints[instLuminosity]) dashboardPayload.append({ "primaryDataset": PD, "release": release, "integratedLuminosity": instLuminosity, "timePerEvent": timePerEvent }) data = "{\"data\":%s}" % str(dashboardPayload).replace("\'", "\"") # let's suppose it works.. testDashBoardPayloadFile = open( os.path.join(getTestBase(), 'WMComponent_t/TaskArchiver_t/DashBoardPayload.json'), 'r') testDashBoardPayload = testDashBoardPayloadFile.read() testDashBoardPayloadFile.close() self.assertEqual(data, testDashBoardPayload) return True
def createConfig(self): """ _createConfig_ Create a config for the JobSubmitter. These parameters are still pulled from the environment. """ config = self.testInit.getConfiguration() self.testInit.generateWorkDir(config) config.component_("Agent") config.Agent.isDocker = False config.section_("JobStateMachine") config.JobStateMachine.couchurl = os.getenv("COUCHURL") config.JobStateMachine.couchDBName = "jobsubmittercaching_t" config.section_("BossAir") config.BossAir.pluginDir = "WMCore.BossAir.Plugins" config.BossAir.pluginNames = ["SimpleCondorPlugin"] config.BossAir.nCondorProcesses = 1 config.component_("JobSubmitter") config.JobSubmitter.submitDir = self.testDir config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') return config
def testTrivialFileCatalog(self): """ Run some simple tests on reading a trivialFileCatalog """ tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T1_US_FNAL_TrivialFileCatalog.xml") if not os.path.exists(tfcFilename): raise Exception("No TrivialFileCatalog found!") tfcInstance = readTFC(tfcFilename) self.assertEqual(type(tfcInstance), type(TrivialFileCatalog())) # Look for similarities in each node of the TFC file for mapping in ['lfn-to-pfn', 'pfn-to-lfn']: for x in tfcInstance[mapping]: self.assertEqual('path-match-expr' in x, True) self.assertEqual('path-match' in x, True) self.assertEqual('protocol' in x, True) self.assertEqual('result' in x, True) self.assertEqual('chain' in x, True) self.assertEqual( x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True, 'Could not find protocol %s' % (x['protocol'])) self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def setUp(self): """ _setUp_ Build a testing environment similar to a WN """ self.testInit = TestInit(__file__) self.testDir = self.testInit.generateWorkDir() # Build a workload/task/step with the basic required information self.workload = newWorkload("UnitTests") self.task = self.workload.newTask("CMSSWExecutor") stepHelper = self.task.makeStep("ExecutorTest") self.step = stepHelper.data template = CMSSWTemplate() template(self.step) self.helper = template.helper(self.step) self.step.application.setup.scramCommand = "scramulator.py" self.step.application.command.executable = "cmsRun.py" self.step.application.setup.scramProject = "CMSSW" self.step.application.setup.scramArch = "slc5_ia32_gcc434" self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z" self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";" self.step.output.jobReport = "FrameworkJobReport.xml" self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus", processedDataset="Test-Era-v1", dataTier="DATA") self.helper.setGlobalTag("Bogus") taskMaker = TaskMaker(self.workload, self.testDir) taskMaker.skipSubscription = True taskMaker.processWorkload() # Build the TaskSpace/StepSpace self.sandboxDir = os.path.join(self.testDir, "UnitTests") self.task.build(self.testDir) sys.path.append(self.testDir) sys.path.append(self.sandboxDir) # Copy the files that cmsRun would have generated in the step space open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close() open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close() shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) # Create a job self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job") self.job["id"] = 1 # Set the PATH binDir = inspect.getsourcefile(ModuleLocator) binDir = binDir.replace("__init__.py", "bin") if not binDir in os.environ['PATH']: os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir) self.oldCwd = os.getcwd()
def testMultiCoreReport(self): """ _testMultiCoreReport_ Verify that multicore reports can be json encoded and uploaded to couch. """ couchdb = CouchServer(os.environ["COUCHURL"]) fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs") self.mcPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl") myReport = Report() myReport.unpersist(self.mcPath) fwjrDocument = { "_id": "303-0", "jobid": 303, "retrycount": 0, "fwjr": myReport.__to_json__(None), "type": "fwjr" } fwjrdatabase.queue(fwjrDocument, timestamp=True) fwjrdatabase.commit() return
def testUploadDownload(self): if 'UFCURL' in os.environ: currdir = getTestBase() upfile = path.join( currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz' ) #file to upload upfileLog = path.join( currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt' ) #file to upload ufc = UserFileCache({ 'endpoint': os.environ['UFCURL'], 'pycurl': True }) #hashkey upload/download res = ufc.upload(upfile) ufc.download(res['hashkey'], output='pippo_publish_down.tgz') #hashkey deletion ufc.removeFile(res['hashkey']) #log upload/download res = ufc.uploadLog(upfileLog) ufc.downloadLog(upfileLog, upfileLog + '.downloaded') self.assertTrue(filecmp.cmp(upfileLog, upfileLog + '.downloaded'))
def testB_ExecuteNonZeroExit(self): """ _ExecuteNonZeroExit_ Test the execution of a script which exits with non-zero code. """ self.step.application.command.executable = "brokenCmsRun.py" shutil.copy(os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"), os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml")) try: os.chdir(self.step.builder.workingDir) executor = StepFactory.getStepExecutor("CMSSW") executor.initialise(self.step, self.job) executor.pre() executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset") try: executor.execute() self.fail("An exception should have been raised") except WMExecutionFailure as ex: executor.diagnostic(ex.code, executor, ExceptionInstance=ex) self.assertEqual(8001, executor.report.getExitCode()) report = Report() report.load("Report.pkl") self.assertEqual(8001, report.getExitCode()) except Exception as ex: self.fail("Failure encountered, %s" % str(ex)) finally: os.chdir(self.oldCwd) return
def testC_testViews(self): """ _testViews_ Prototype test for what should be a lot of other tests. """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.attachments['attach1'] = attach configCache.document['md5_hash'] = "somemd5" psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.save() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache2.document['md5_hash'] = configCache.document['md5_hash'] configCache2.load() self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache2.delete() return
def testB_addingConfigsAndTweaks(self): """ _addingConfigsAndTweaks_ Test adding config files and tweak files """ PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you." attach = "Hello, I am an attachment" configCache = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test') configCache.createUserGroup(groupname="testGroup", username='******') configCache.setPSetTweaks(PSetTweak=PSetTweak) configCache.attachments['attach1'] = attach psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt") configCache.addConfig(newConfig=psetPath, psetHash=None) configCache.setLabel("sample-label") configCache.setDescription("describe this config here") configCache.save() configString1 = configCache.getConfig() configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName='config_test', id=configCache.getCouchID(), rev=configCache.getCouchRev()) configCache2.loadByID(configCache.getCouchID()) configString2 = configCache2.getConfig() self.assertEqual(configString1, configString2) self.assertEqual(configCache2.attachments.get('attach1', None), attach) configCache.delete() return
def testFNALSiteLocalConfig(self): """ _testFNALSiteLocalConfig_ Verify that the FNAL site config file is parsed correctly. """ fnalConfigFileName = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T1_US_FNAL_SiteLocalConfig.xml") mySiteConfig = SiteLocalConfig(fnalConfigFileName) assert mySiteConfig.siteName == "T1_US_FNAL", \ "Error: Wrong site name." assert len(mySiteConfig.eventData.keys()) == 1, \ "Error: Wrong number of event data keys." assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \ "Eroror: Event data catalog is wrong." goldenServers = [ "http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier1.cern.ch:8000/FrontierInt", "http://cmsfrontier2.cern.ch:8000/FrontierInt", "http://cmsfrontier3.cern.ch:8000/FrontierInt", "http://cmsfrontier4.cern.ch:8000/FrontierInt" ] for frontierServer in mySiteConfig.frontierServers: assert frontierServer in goldenServers, \ "Error: Unknown server: %s" % frontierServer goldenServers.remove(frontierServer) assert len(goldenServers) == 0, \ "Error: Missing frontier servers." goldenProxies = [ "http://cmsfrontier1.fnal.gov:3128", "http://cmsfrontier2.fnal.gov:3128", "http://cmsfrontier3.fnal.gov:3128", "http://cmsfrontier4.fnal.gov:3128" ] for frontierProxy in mySiteConfig.frontierProxies: assert frontierProxy in goldenProxies, \ "Error: Unknown proxy: %s" % frontierProxy goldenProxies.remove(frontierProxy) assert len(goldenProxies) == 0, \ "Error: Missing proxy servers." assert mySiteConfig.localStageOut["se-name"] == "cmssrm.fnal.gov", \ "Error: Wrong se name from local stageout." assert mySiteConfig.localStageOut["command"] == "dccp-fnal", \ "Error: Wrong stage out command." assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=srmv2", \ "Error: TFC catalog is not correct." assert mySiteConfig.fallbackStageOut == [], \ "Error: Fallback config is incorrect." return
def testVanderbiltSiteLocalConfig(self): """ _testFNALSiteLocalConfig_ Verify that the FNAL site config file is parsed correctly. """ vandyConfigFileName = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T3_US_Vanderbilt_SiteLocalConfig.xml") mySiteConfig = SiteLocalConfig(vandyConfigFileName) assert mySiteConfig.siteName == "T3_US_Vanderbilt", \ "Error: Wrong site name." assert len(mySiteConfig.eventData.keys()) == 1, \ "Error: Wrong number of event data keys." assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=direct", \ "Eroror: Event data catalog is wrong." goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier1.cern.ch:8000/FrontierInt", "http://cmsfrontier2.cern.ch:8000/FrontierInt", "http://cmsfrontier3.cern.ch:8000/FrontierInt"] for frontierServer in mySiteConfig.frontierServers: assert frontierServer in goldenServers, \ "Error: Unknown server: %s" % frontierServer goldenServers.remove(frontierServer) assert len(goldenServers) == 0, \ "Error: Missing frontier servers." goldenProxies = ["http://se1.accre.vanderbilt.edu:3128"] for frontierProxy in mySiteConfig.frontierProxies: assert frontierProxy in goldenProxies, \ "Error: Unknown proxy: %s" % frontierProxy goldenProxies.remove(frontierProxy) assert len(goldenProxies) == 0, \ "Error: Missing proxy servers." assert mySiteConfig.localStageOut["se-name"] == "se1.accre.vanderbilt.edu", \ "Error: Wrong se name from local stageout." assert mySiteConfig.localStageOut["command"] == "srmv2", \ "Error: Wrong stage out command." assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=srmv2", \ "Error: TFC catalog is not correct." assert len(mySiteConfig.fallbackStageOut) == 1, \ "Error: Incorrect number of fallback stageout methods" assert mySiteConfig.fallbackStageOut[0]["command"] == "srmv2-lcg", \ "Error: Incorrect fallback command." assert mySiteConfig.fallbackStageOut[0]["se-name"] == "se1.accre.vanderbilt.edu", \ "Error: Incorrect fallback SE." assert mySiteConfig.fallbackStageOut[0]["lfn-prefix"] == "srm://se1.accre.vanderbilt.edu:6288/srm/v2/server?SFN=", \ "Error: Incorrect fallback LFN prefix." return
def setUp(self): """ _setUp_ Figure out the location of the XML report produced by CMSSW. """ self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("report_t/fwjrs", "FWJRDump") self.xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml") self.badxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml") self.skippedFilesxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml") self.skippedAllFilesxmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml") self.fallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml") self.twoFileFallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml") self.pileupXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWPileup.xml") self.testDir = self.testInit.generateWorkDir() return
def testMultipleInputs(self): """ _testMultipleInputs_ Verify that parsing XML reports with multiple inputs works correctly. """ xmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWMultipleInput.xml") myReport = Report("cmsRun1") myReport.parse(xmlPath) assert hasattr(myReport.data.cmsRun1.input, "source"), \ "Error: Report missing input source." inputFiles = myReport.getInputFilesFromStep("cmsRun1") assert len(inputFiles) == 2, \ "Error: Wrong number of input files." for inputFile in inputFiles: assert inputFile["input_type"] == "primaryFiles", \ "Error: Wrong input type." assert inputFile["module_label"] == "source", \ "Error: Module label is wrong" assert inputFile["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \ "Error: Catalog is wrong." assert inputFile["events"] == 2, \ "Error: Wrong number of events." assert inputFile["input_source_class"] == "PoolSource", \ "Error: Wrong input source class." if inputFile["guid"] == "F0875ECD-3347-DF11-9FE0-003048678A80": assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \ "Error: Input LFN is wrong." assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \ "Error: Input PFN is wrong." assert len(inputFile["runs"]) == 1, \ "Error: Wrong number of runs." assert list(inputFile["runs"])[0].run == 124216, \ "Error: Wrong run number." assert 1 in list(inputFile["runs"])[0], \ "Error: Wrong lumi sections in input file." else: assert inputFile["guid"] == "626D74CE-3347-DF11-9363-0030486790C0", \ "Error: Wrong guid." assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \ "Error: Input LFN is wrong." assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \ "Error: Input PFN is wrong." assert len(inputFile["runs"]) == 1, \ "Error: Wrong number of runs." assert list(inputFile["runs"])[0].run == 124216, \ "Error: Wrong run number." assert 2 in list(inputFile["runs"])[0], \ "Error: Wrong lumi sections in input file." return
def setUp(self): """ _setUp_ Setup a dashboard reporter """ self.reporter = DashboardReporter(config=None) self.processingReport = ProcessingSample.report self.mergeReport = MergeSample.report self.errorReport = ErrorSample.report self.fallbackReport = FallbackSample.report self.twoFileFallbackXmlPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml") self.pileupXmlPath = os.path.join( getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWPileup.xml") return
def testCreateStageOutCommand_realFile(self): base = os.path.join(getTestBase(), "WMCore_t/Storage_t/ExecutableCommands.py") result = self.CPImpl.createStageOutCommand(base, "test") expectedResult = [ " '590' -eq $DEST_SIZE", "DEST_SIZE=`/bin/ls -l test", base ] for text in expectedResult: self.assertIn(text, result)
def testFNALSiteLocalConfig(self): """ _testFNALSiteLocalConfig_ Verify that the FNAL site config file is parsed correctly. """ fnalConfigFileName = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T1_US_FNAL_SiteLocalConfig.xml") mySiteConfig = SiteLocalConfig(fnalConfigFileName) assert mySiteConfig.siteName == "T1_US_FNAL", \ "Error: Wrong site name." assert len(mySiteConfig.eventData.keys()) == 1, \ "Error: Wrong number of event data keys." assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \ "Eroror: Event data catalog is wrong." goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier.cern.ch:8000/FrontierInt", "http://cmsfrontier1.cern.ch:8000/FrontierInt", "http://cmsfrontier2.cern.ch:8000/FrontierInt", "http://cmsfrontier3.cern.ch:8000/FrontierInt", "http://cmsfrontier4.cern.ch:8000/FrontierInt"] for frontierServer in mySiteConfig.frontierServers: assert frontierServer in goldenServers, \ "Error: Unknown server: %s" % frontierServer goldenServers.remove(frontierServer) assert len(goldenServers) == 0, \ "Error: Missing frontier servers." goldenProxies = ["http://cmsfrontier1.fnal.gov:3128", "http://cmsfrontier2.fnal.gov:3128", "http://cmsfrontier3.fnal.gov:3128", "http://cmsfrontier4.fnal.gov:3128"] for frontierProxy in mySiteConfig.frontierProxies: assert frontierProxy in goldenProxies, \ "Error: Unknown proxy: %s" % frontierProxy goldenProxies.remove(frontierProxy) assert len(goldenProxies) == 0, \ "Error: Missing proxy servers." assert mySiteConfig.localStageOut["se-name"] == "cmssrm.fnal.gov", \ "Error: Wrong se name from local stageout." assert mySiteConfig.localStageOut["command"] == "dccp-fnal", \ "Error: Wrong stage out command." assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=srmv2", \ "Error: TFC catalog is not correct." assert mySiteConfig.fallbackStageOut == [], \ "Error: Fallback config is incorrect." return
def testMultipleRegexMatch(self): # Check that an lfn is converted to the right pfn in_lfn = '/store/user/fred/data' tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T2_CH_CERNBOX_TrivialFileCatalog.xml") tfc = readTFC(tfc_file) out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data" pfn = tfc.matchLFN('srmv2', in_lfn) self.assertEqual(out_pfn, pfn) tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T2_PT_NCG_Lisbon_TrivialFileCatalog.xml") tfc = readTFC(tfc_file) out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data" pfn = tfc.matchLFN('srmv2', in_lfn) self.assertEqual(out_pfn, pfn) tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T2_US_Florida_TrivialFileCatalog.xml") tfc = readTFC(tfc_file) out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data" pfn = tfc.matchLFN('srmv2', in_lfn) self.assertEqual(out_pfn, pfn) tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T2_ES_IFCA_TrivialFileCatalog.xml") tfc = readTFC(tfc_file) out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data" pfn = tfc.matchLFN('srmv2', in_lfn) self.assertEqual(out_pfn, pfn) tfc_file = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T2_US_Nebraska_TrivialFileCatalog.xml") tfc = readTFC(tfc_file) out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data" pfn = tfc.matchLFN('srmv2', in_lfn) self.assertEqual(out_pfn, pfn)
def testExceptionFilter(self): """ Test getWMExceptionStr function. """ count = 0 logPath = os.path.join(getTestBase(), "WMCore_t/test_condor.log") for result in listWMExceptionStr(logPath): # print(result) count += 1 self.assertEqual(count, 4)
def setupJobEnvironment(self, name = 'test'): """ _setupJobEnvironment_ Make some sort of environment in which to run tests """ os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T1_US_FNAL_SiteLocalConfig.xml") return
def setUp(self): filePath = os.path.join(getTestBase(), "WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json") with open(filePath, "r") as f: gqData = JsonWrapper.load(f) self.gqElements = [] for ele in gqData: self.gqElements.append(WorkQueueElement(**ele))
def testChecksum(self): """ Tests checksum method """ checksum1 = calculateChecksum( tarfile_=path.join( getTestBase(), "WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz" ) ) checksum2 = calculateChecksum( tarfile_=path.join( getTestBase(), "WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz", ) ) self.assertTrue(checksum1) self.assertTrue(checksum2) self.assertFalse(checksum1 == checksum2) self.assertRaises(IOError, calculateChecksum, **{"tarfile_": "does_not_exist"}) return
def testUploadDownload(self): if "UFCURL" in os.environ: currdir = getTestBase() upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload ufc = UserFileCache({"endpoint": os.environ["UFCURL"]}) # named upload/download res = ufc.upload(upfile, "name_publish.tgz") ufc.download(name=res["name"], output="name_publish.tgz") # hashkey upload/download res = ufc.upload(upfile) ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
def testUploadDownload(self): if 'UFCURL' in os.environ: currdir = getTestBase() upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload ufc = UserFileCache({'endpoint':os.environ['UFCURL']}) #named upload/download res = ufc.upload(upfile, 'name_publish.tgz') ufc.download(name=res['name'], output='name_publish.tgz') #hashkey upload/download res = ufc.upload(upfile) ufc.download(res['hashkey'], output='pippo_publish_down.tgz')