def testB_TestBase(self):
"""
_TestBase_
See if we can use the defaults or an environment variable
to build a testBase
NOTE: This has to run in WMCore/test/python/WMCore_t, and
so is listed as integration.
"""
test = os.path.normpath(
os.path.join(os.getcwd(), '../../../test/python'))
base = getTestBase()
self.assertEqual(base, test)
base = getTestBase(importFlag=False)
self.assertEqual(base, test)
os.environ['WMCORE_TEST_ROOT'] = '/thisdirectoryshouldneverexist'
base = getTestBase(importFlag=False)
self.assertEqual(base, test)
os.environ['WMCORE_TEST_ROOT'] = '/tmp'
base = getTestBase(importFlag=False)
self.assertEqual(base, '/tmp')
return
def testGetIterMatchObjectOnRegex(self):
logPath = os.path.join(getTestBase(), "WMCore_t/test_empty.log")
for mo in getIterMatchObjectOnRegexp(logPath, WMEXCEPTION_REGEXP):
pass
count = 0
ecount = 0
logPath = os.path.join(getTestBase(), "WMCore_t/test_condor.log")
for mo in getIterMatchObjectOnRegexp(logPath, WMEXCEPTION_REGEXP):
errMsg = mo.group("WMException")
if errMsg:
count += 1
error = mo.group("ERROR")
if error:
ecount += 1
self.assertEqual(count, 4)
self.assertEqual(ecount, 1)
rcount = 0
scount = 0
for mo in getIterMatchObjectOnRegexp(logPath,
CONDOR_LOG_FILTER_REGEXP):
if mo.group("Reason"):
reason = mo.group("Reason")
rcount += 1
if mo.group("Site"):
site = mo.group("Site")
scount += 1
self.assertEqual(rcount, 1)
self.assertEqual(scount, 2)
self.assertEqual(site, 'T1_US_FNAL')
self.assertEqual(reason, 'via condor_rm (by user cmst1)')
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
#Here Test the CondorPlugin instead of MockPlugin
#config.BossAir.pluginNames = ['CondorPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', "submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
config.JobSubmitter.componentDir = os.path.join(
self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def getConfig(self):
"""
_getConfig_
Gets a basic config from default location
"""
config = Configuration()
config.component_("Agent")
config.Agent.WMSpecDirectory = self.testDir
config.Agent.agentName = 'testAgent'
config.Agent.componentName = self.componentName
config.Agent.useHeartbeat = False
#First the general stuff
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", self.testDir)
#Now the CoreDatabase information
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
# BossAir and MockPlugin configuration
config.section_("BossAir")
config.BossAir.pluginNames = ['MockPlugin']
config.BossAir.pluginDir = 'WMCore.BossAir.Plugins'
config.BossAir.multicoreTaskTypes = ['MultiProcessing', 'MultiProduction']
config.BossAir.nCondorProcesses = 1
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
"submit.sh")
# JobSubmitter configuration
config.component_("JobSubmitter")
config.JobSubmitter.logLevel = 'DEBUG'
config.JobSubmitter.maxThreads = 1
config.JobSubmitter.pollInterval = 10
config.JobSubmitter.submitScript = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
'submit.sh')
config.JobSubmitter.componentDir = os.path.join(self.testDir, 'Components')
config.JobSubmitter.workerThreads = 2
config.JobSubmitter.jobsPerWorker = 200
#JobStateMachine
config.component_('JobStateMachine')
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "jobsubmitter_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
# Needed, because this is a test
os.makedirs(config.JobSubmitter.componentDir)
return config
def testChecksum(self):
"""
Tests checksum method
"""
checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), '../data/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'})
return
def testChecksum(self):
"""
Tests checksum method
"""
self.ufc = UserFileCache()
checksum1 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz'))
checksum2 = self.ufc.checksum(fileName=path.join(getTestBase(), 'WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz'))
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, self.ufc.checksum, **{'fileName': 'does_not_exist'})
return
def stuffWMBS(self):
"""
_stuffWMBS_
Inject the workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def stuffWMBS(self):
"""
_stuffWMBS_
Inject the workflow in WMBS and add the subscriptions
"""
testWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask", owner_vogroup = "", owner_vorole = "")
testWorkflow.create()
testMergeWorkflow = Workflow(spec = os.path.join(getTestBase(),
"WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl"),
owner = "/CN=OU/DN=SomeoneWithPermissions",
name = "BogusRequest", task = "BogusTask/Merge", owner_vogroup = "", owner_vorole = "")
testMergeWorkflow.create()
testWMBSFileset = Fileset(name = "TopFileset")
testWMBSFileset.create()
testWMBSFilesetUnmerged = Fileset(name = "UnmergedFileset")
testWMBSFilesetUnmerged.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFilesetUnmerged.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFilesetUnmerged.commit()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testSubscriptionMerge = Subscription(fileset = testWMBSFilesetUnmerged,
workflow = testMergeWorkflow,
type = "Merge")
testSubscriptionMerge.create()
return (testSubscription, testSubscriptionMerge)
def _configCacheId(self, label):
"""Return config cache id for given config label"""
key, cert = self.__class__.reqmgr['requests'].getKeyCert()
configCache = ConfigCache(self.__class__.endpoint + '/couchdb',
'reqmgr_config_cache',
ckey=key,
cert=cert)
try:
configCacheId = configCache.getIDFromLabel(label)
except:
configCacheId = None
if configCacheId:
return configCacheId
# The following will fail if FWCore.ParameterSet not in PYTHONPATH
from PSetTweaks.WMTweak import makeTweak
configCache.createUserGroup('test', 'test')
configDir = os.path.join(getTestBase(), '..', '..', 'test', 'data',
'configs')
configCache.addConfig(os.path.join(configDir, label + '.py'))
configCache.setLabel(label)
configCache.setDescription(label)
modPath = imp.find_module(label, [configDir])
loadedConfig = imp.load_module(label, modPath[0], modPath[1],
modPath[2])
configCache.setPSetTweaks(
makeTweak(loadedConfig.process).jsondictionary())
configCache.save()
return configCache.getIDFromLabel(label)
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test',
id = configCache.getCouchID(),
rev = configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache.createUserGroup(groupname = "testGroup", username = '******')
configCache.setPSetTweaks(PSetTweak = PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig = psetPath, psetHash = None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"], couchDBName = 'config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def testTrivialFileCatalog(self):
"""
Run some simple tests on reading a trivialFileCatalog
"""
tfcFilename = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
if not os.path.exists(tfcFilename):
raise Exception("No TrivialFileCatalog found!")
tfcInstance = readTFC(tfcFilename)
self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))
# Look for similarities in each node of the TFC file
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for x in tfcInstance[mapping]:
self.assertEqual('path-match-expr' in x, True)
self.assertEqual('path-match' in x, True)
self.assertEqual('protocol' in x, True)
self.assertEqual('result' in x, True)
self.assertEqual('chain' in x, True)
self.assertEqual(x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'],
True, 'Could not find protocol %s' % (x['protocol']))
self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def setUp(self):
"""
_setUp_
Setup a dashboard reporter
"""
self.reporter = DashboardReporter(config=None)
self.processingReport = ProcessingSample.report
self.mergeReport = MergeSample.report
self.errorReport = ErrorSample.report
self.fallbackReport = FallbackSample.report
self.twoFileFallbackXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml")
self.pileupXmlPath = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWPileup.xml")
return
def testBasic(self):
"""
_testBasic_
Test the basic functions of the DBSBufferDataset,
create, load, exists and also the ability
to add subscriptions.
"""
originalDataset = DBSBufferDataset(path = '/bogus/bogus/go')
originalDataset.create()
myThread = threading.currentThread()
result = myThread.dbi.processData("SELECT id FROM dbsbuffer_dataset")[0].fetchall()
self.assertEqual(originalDataset.exists(), result[0][0])
duplicateDataset = DBSBufferDataset(path = '/bogus/bogus/go')
duplicateDataset.create()
self.assertEqual(originalDataset.exists(), duplicateDataset.exists())
result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset")[0].fetchall()
self.assertEqual(result[0][0], 1)
loadedDataset = DBSBufferDataset(path = '/bogus/bogus/go')
loadedDataset.load()
self.assertEqual(loadedDataset.exists(), originalDataset.exists())
secondDataset = DBSBufferDataset(path = '/BogusPrimary/Run2012Z-PromptReco-v1/RECO')
secondDataset.create()
workload = WMWorkloadHelper()
workload.load(os.path.join(getTestBase(), 'WMComponent_t/PhEDExInjector_t/specs/TestWorkload.pkl'))
secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO'])
secondDataset.addSubscription(workload.getSubscriptionInformation()['/BogusPrimary/Run2012Z-PromptReco-v1/RECO'])
self.assertEqual(len(secondDataset['subscriptions']), 3)
result = myThread.dbi.processData("SELECT COUNT(id) FROM dbsbuffer_dataset_subscription")[0].fetchall()
self.assertEqual(result[0][0], 3)
return
def testGetIterMatchObjectOnRegex(self):
count = 0
ecount = 0
logPath = os.path.join(getTestBase(), "WMCore_t/test_condor.log")
for mo in getIterMatchObjectOnRegexp(logPath, WMEXCEPTION_REGEXP):
errMsg = mo.group("WMException")
if errMsg:
count += 1
error = mo.group("ERROR")
if error:
ecount += 1
self.assertEqual(count, 4)
self.assertEqual(ecount, 1)
rcount = 0
scount = 0
for mo in getIterMatchObjectOnRegexp(logPath, CONDOR_LOG_FILTER_REGEXP):
if mo.group("Reason"):
reason = mo.group("Reason")
rcount += 1
if mo.group("Site"):
site = mo.group("Site")
scount += 1
self.assertEqual(rcount, 1)
self.assertEqual(scount, 2)
self.assertEqual(site, 'T1_US_FNAL')
self.assertEqual(reason, 'via condor_rm (by user cmst1)')
def createConfig(self):
"""
_createConfig_
Create a config for the JobSubmitter. These parameters are still pulled
from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "jobsubmittercaching_t"
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["SimpleCondorPlugin"]
config.BossAir.nCondorProcesses = 1
config.component_("JobSubmitter")
config.JobSubmitter.submitDir = self.testDir
config.JobSubmitter.submitScript = os.path.join(getTestBase(),
'WMComponent_t/JobSubmitter_t',
'submit.sh')
return config
def testPerformanceJSON(self):
"""
_testPerformanceJSON_
Verify that the performance section of the report is correctly converted
to JSON.
"""
xmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/PerformanceReport.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
perfSection = myReport.__to_json__(thunker = None)["steps"]["cmsRun1"]["performance"]
self.assertTrue(perfSection.has_key("storage"),
"Error: Storage section is missing.")
self.assertTrue(perfSection.has_key("memory"),
"Error: Memory section is missing.")
self.assertTrue(perfSection.has_key("cpu"),
"Error: CPU section is missing.")
self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105",
"Error: AvgEventCPU is wrong.")
self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703",
"Error: TotalJobTime is wrong.")
self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166,
"Error: readTotalMB is wrong.")
self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653,
"Error: readMaxMSec is wrong")
self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293",
"Error: PeakValueRss is wrong.")
self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281",
"Error: PeakValueVsize is wrong.")
return
def testreportFailureToDashboard(self):
"""
_testreportFailureToDashboard_
test report Failure to dashboard with/without job_ad file
"""
self.assertEqual(0, reportFailureToDashboard(0, None))
# parseAd is called by reportFailureToDashboard, but if we would
# call it from reportFailureToDashboard error will be suppressed
# and it will return same exit code
# So lets call it directly
if '_CONDOR_JOB_AD' in os.environ:
del os.environ["_CONDOR_JOB_AD"]
self.assertRaises(KeyError, parseAd)
self.assertEqual(0, reportFailureToDashboard(0))
os.environ["_CONDOR_JOB_AD"] = "testFile"
# if this environment variable is set, it will check this file
# if file is not present, it will raise IOError
self.assertRaises(IOError, parseAd)
# Test real job_ad file and check if it returns dict.
os.environ["_CONDOR_JOB_AD"] = os.path.join(getTestBase(), '..',
'data', 'WMCore',
'Services', 'Dashboard',
'job_ad_file')
out = parseAd()
self.assertTrue(isinstance(out, dict))
# Also test it sending directly parsed job ad
self.assertEqual(0, reportFailureToDashboard(0, out))
def testB_ExecuteNonZeroExit(self):
"""
_ExecuteNonZeroExit_
Test the execution of a script
which exits with non-zero code.
"""
self.step.application.command.executable = "brokenCmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure, ex:
executor.diagnostic(ex.code, executor, ExceptionInstance = ex)
self.assertEqual(8001, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(8001, report.getExitCode())
except Exception, ex:
self.fail("Failure encountered, %s" % str(ex))
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testInsertT0(self):
"""
_testInsertT0_
Test to see if we can insert the Tier-0 alone
with a single option
"""
self.createConfig()
resControlPath = os.path.join(getTestBase(), "../../bin/wmagent-resource-control")
env = os.environ
env['PYTHONPATH'] = ":".join(sys.path)
cmdline = [resControlPath, "--add-T0" ]
retval = subprocess.Popen( cmdline,
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT,
env = env)
(_, _) = retval.communicate()
myResourceControl = ResourceControl()
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(result), 1)
self.assertTrue('CERN' in result)
for x in result:
self.assertEqual(len(result[x]['thresholds']), 9)
self.assertEqual(result[x]['total_pending_slots'], 500)
self.assertEqual(result[x]['total_running_slots'], -1)
for taskType, thresh in result[x]['thresholds'].items():
if taskType == 'Processing':
self.assertEqual(thresh['priority'], 0)
self.assertEqual(thresh['max_slots'], -1)
# Verify that sites with more than one SE were added correctly.
cernInfo = myResourceControl.listSiteInfo("CERN")
self.assertTrue(len(cernInfo["pnn"]) == 2)
return
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["MockPlugin"]
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
"MergeSuccess.pkl")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return config
def testD_ExecuteNoOutput(self):
"""
_ExecuteNoOutput_
Test what happens when no output is produced,
the proper error should be included.
"""
self.step.application.command.executable = "cmsRun.py"
shutil.copy(
os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
os.path.join(self.step.builder.workingDir,
"FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
self.assertEqual(60450, executor.report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testInsertSite(self):
"""
_testInsertSite_
Test to see if we can insert a fake test site alone
with a single option
"""
self.createConfig()
resControlPath = os.path.join(getTestBase(), "../../bin/wmagent-resource-control")
env = os.environ
env['PYTHONPATH'] = ":".join(sys.path)
cmdline = [resControlPath, "--add-Test"]
retval = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
(_, _) = retval.communicate()
myResourceControl = ResourceControl()
result = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(result), 1)
self.assertTrue('CERN' in result)
for x in result:
self.assertEqual(len(result[x]['thresholds']), 7)
self.assertEqual(result[x]['total_pending_slots'], 500)
self.assertEqual(result[x]['total_running_slots'], 1)
for taskType, thresh in result[x]['thresholds'].items():
if taskType == 'Processing':
self.assertEqual(thresh['priority'], 0)
self.assertEqual(thresh['max_slots'], 1)
# Verify that sites with more than one SE were added correctly.
cernInfo = myResourceControl.listSiteInfo("CERN")
self.assertTrue(len(cernInfo["pnn"]) == 2)
return
def testJSONEncoding(self):
"""
_testJSONEncoding_
Verify that turning the FWJR into a JSON object works correctly.
"""
xmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
jsonReport = myReport.__to_json__(None)
assert "task" in jsonReport.keys(), \
"Error: Task name missing from report."
assert len(jsonReport["steps"].keys()) == 1, \
"Error: Wrong number of steps in report."
assert "cmsRun1" in jsonReport["steps"].keys(), \
"Error: Step missing from json report."
cmsRunStep = jsonReport["steps"]["cmsRun1"]
jsonReportSections = ["status", "errors", "logs", "parameters", "site",
"analysis", "cleanup", "input", "output", "start"]
for jsonReportSection in jsonReportSections:
assert jsonReportSection in cmsRunStep.keys(), \
"Error: missing section: %s" % jsonReportSection
return
def testD_ExecuteNoOutput(self):
"""
_ExecuteNoOutput_
Test what happens when no output is produced,
the proper error should be included.
"""
self.step.application.command.executable = "cmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
executor.execute()
executor.post()
self.assertEqual(60450, executor.report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testPerformanceJSON(self):
"""
_testPerformanceJSON_
Verify that the performance section of the report is correctly converted
to JSON.
"""
xmlPath = os.path.join(
getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
perfSection = myReport.__to_json__(
thunker=None)["steps"]["cmsRun1"]["performance"]
self.assertTrue("storage" in perfSection,
"Error: Storage section is missing.")
self.assertTrue("memory" in perfSection,
"Error: Memory section is missing.")
self.assertTrue("cpu" in perfSection, "Error: CPU section is missing.")
self.assertEqual(perfSection["cpu"]["AvgEventCPU"], "0.626105",
"Error: AvgEventCPU is wrong.")
self.assertEqual(perfSection["cpu"]["TotalJobTime"], "23.5703",
"Error: TotalJobTime is wrong.")
self.assertEqual(perfSection["storage"]["readTotalMB"], 39.6166,
"Error: readTotalMB is wrong.")
self.assertEqual(perfSection["storage"]["readMaxMSec"], 320.653,
"Error: readMaxMSec is wrong")
self.assertEqual(perfSection["memory"]["PeakValueRss"], "492.293",
"Error: PeakValueRss is wrong.")
self.assertEqual(perfSection["memory"]["PeakValueVsize"], "643.281",
"Error: PeakValueVsize is wrong.")
return
def testreportFailureToDashboard(self):
"""
_testreportFailureToDashboard_
test report Failure to dashboard with/without job_ad file
"""
self.assertEqual(0, reportFailureToDashboard(0, None))
# parseAd is called by reportFailureToDashboard, but if we would
# call it from reportFailureToDashboard error will be suppressed
# and it will return same exit code
# So lets call it directly
if '_CONDOR_JOB_AD' in os.environ:
del os.environ["_CONDOR_JOB_AD"]
self.assertRaises(KeyError, parseAd)
self.assertEqual(0, reportFailureToDashboard(0))
os.environ["_CONDOR_JOB_AD"] = "testFile"
# if this environment variable is set, it will check this file
# if file is not present, it will raise IOError
self.assertRaises(IOError, parseAd)
# Test real job_ad file and check if it returns dict.
os.environ["_CONDOR_JOB_AD"] = os.path.join(getTestBase(), '..', 'data', 'WMCore', 'Services', 'Dashboard', 'job_ad_file')
out = parseAd()
self.assertTrue(isinstance(out, dict))
# Also test it sending directly parsed job ad
self.assertEqual(0, reportFailureToDashboard(0, out))
def testJSONEncoding(self):
"""
_testJSONEncoding_
Verify that turning the FWJR into a JSON object works correctly.
"""
xmlPath = os.path.join(
getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
jsonReport = myReport.__to_json__(None)
assert "task" in jsonReport.keys(), \
"Error: Task name missing from report."
assert len(jsonReport["steps"].keys()) == 1, \
"Error: Wrong number of steps in report."
assert "cmsRun1" in jsonReport["steps"].keys(), \
"Error: Step missing from json report."
cmsRunStep = jsonReport["steps"]["cmsRun1"]
jsonReportSections = [
"status", "errors", "logs", "parameters", "site", "analysis",
"cleanup", "input", "output", "start"
]
for jsonReportSection in jsonReportSections:
assert jsonReportSection in cmsRunStep.keys(), \
"Error: missing section: %s" % jsonReportSection
return
def testPerformanceSummary(self):
"""
_testPerformanceSummary_
Test whether or not we can pull performance information
out of a Timing/SimpleMemoryCheck jobReport
"""
xmlPath = os.path.join(
getTestBase(), "WMCore_t/FwkJobReport_t/PerformanceReport.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
# Do a brief check of the three sections
perf = myReport.data.cmsRun1.performance
self.assertEqual(perf.memory.PeakValueRss, '492.293')
self.assertEqual(perf.cpu.TotalJobCPU, '9.16361')
self.assertEqual(perf.storage.writeTotalMB, 5.22226)
self.assertAlmostEqual(perf.storage.writeTotalSecs, 0,
places=0) # actual value is 0.06
self.assertEqual(perf.storage.readPercentageOps, 0.98585512216030857)
return
def publishPerformanceDashBoard(self, dashBoardUrl, PD, release, worthPoints):
dashboardPayload = []
for instLuminosity in worthPoints:
timePerEvent = int(worthPoints[instLuminosity])
dashboardPayload.append(
{
"primaryDataset": PD,
"release": release,
"integratedLuminosity": instLuminosity,
"timePerEvent": timePerEvent,
}
)
data = '{"data":%s}' % str(dashboardPayload).replace("'", '"')
# let's suppose it works..
testDashBoardPayloadFile = open(
os.path.join(getTestBase(), "WMComponent_t/TaskArchiver_t/DashBoardPayload.json"), "r"
)
testDashBoardPayload = testDashBoardPayloadFile.read()
testDashBoardPayloadFile.close()
self.assertEquals(data, testDashBoardPayload)
return True
def setUp(self):
"""
_setUp_
Figure out the location of the XML report produced by CMSSW.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")
testData = os.path.join(getTestBase(), "WMCore_t/FwkJobReport_t")
self.xmlPath = os.path.join(testData, "CMSSWProcessingReport.xml")
self.badxmlPath = os.path.join(testData, "CMSSWFailReport2.xml")
self.skippedFilesxmlPath = os.path.join(
testData, "CMSSWSkippedNonExistentFile.xml")
self.skippedAllFilesxmlPath = os.path.join(testData,
"CMSSWSkippedAll.xml")
self.fallbackXmlPath = os.path.join(testData, "CMSSWInputFallback.xml")
self.twoFileFallbackXmlPath = os.path.join(testData,
"CMSSWTwoFileRemote.xml")
self.pileupXmlPath = os.path.join(testData, "CMSSWPileup.xml")
self.withEventsXmlPath = os.path.join(testData,
"CMSSWWithEventCounts.xml")
self.testDir = self.testInit.generateWorkDir()
return
def publishPerformanceDashBoard(self, dashBoardUrl, PD, release,
worthPoints):
dashboardPayload = []
for instLuminosity in worthPoints:
timePerEvent = int(worthPoints[instLuminosity])
dashboardPayload.append({
"primaryDataset": PD,
"release": release,
"integratedLuminosity": instLuminosity,
"timePerEvent": timePerEvent
})
data = "{\"data\":%s}" % str(dashboardPayload).replace("\'", "\"")
# let's suppose it works..
testDashBoardPayloadFile = open(
os.path.join(getTestBase(),
'WMComponent_t/TaskArchiver_t/DashBoardPayload.json'),
'r')
testDashBoardPayload = testDashBoardPayloadFile.read()
testDashBoardPayloadFile.close()
self.assertEqual(data, testDashBoardPayload)
return True
def createConfig(self):
"""
_createConfig_
Create a config and save it to the temp dir. Set the WMAGENT_CONFIG
environment variable so the config gets picked up.
"""
config = Configuration()
config.section_("General")
config.General.workDir = os.getenv("TESTDIR", os.getcwd())
config.section_("Agent")
config.Agent.componentName = "resource_control_t"
config.section_("CoreDatabase")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.socket = os.getenv("DBSOCK")
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv('COUCHURL')
config.JobStateMachine.couchDBName = "bossair_t"
config.JobStateMachine.jobSummaryDBName = 'wmagent_summary_t'
config.JobStateMachine.summaryStatsDBName = 'stat_summary_t'
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["MockPlugin"]
config.BossAir.section_("MockPlugin")
config.BossAir.MockPlugin.fakeReport = os.path.join(getTestBase(),
'WMComponent_t/JobAccountant_t/fwjrs',
"MergeSuccess.pkl")
configHandle = open(os.path.join(self.tempDir, "config.py"), "w")
configHandle.write(str(config))
configHandle.close()
os.environ["WMAGENT_CONFIG"] = os.path.join(self.tempDir, "config.py")
return config
def createConfig(self):
"""
_createConfig_
Create a config for the JobSubmitter. These parameters are still pulled
from the environment.
"""
config = self.testInit.getConfiguration()
self.testInit.generateWorkDir(config)
config.component_("Agent")
config.Agent.isDocker = False
config.section_("JobStateMachine")
config.JobStateMachine.couchurl = os.getenv("COUCHURL")
config.JobStateMachine.couchDBName = "jobsubmittercaching_t"
config.section_("BossAir")
config.BossAir.pluginDir = "WMCore.BossAir.Plugins"
config.BossAir.pluginNames = ["SimpleCondorPlugin"]
config.BossAir.nCondorProcesses = 1
config.component_("JobSubmitter")
config.JobSubmitter.submitDir = self.testDir
config.JobSubmitter.submitScript = os.path.join(
getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh')
return config
def testTrivialFileCatalog(self):
"""
Run some simple tests on reading a trivialFileCatalog
"""
tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
if not os.path.exists(tfcFilename):
raise Exception("No TrivialFileCatalog found!")
tfcInstance = readTFC(tfcFilename)
self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))
# Look for similarities in each node of the TFC file
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for x in tfcInstance[mapping]:
self.assertEqual('path-match-expr' in x, True)
self.assertEqual('path-match' in x, True)
self.assertEqual('protocol' in x, True)
self.assertEqual('result' in x, True)
self.assertEqual('chain' in x, True)
self.assertEqual(
x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True,
'Could not find protocol %s' % (x['protocol']))
self.assertEqual(x['chain'], None,
'Invalid chain %s' % (x['chain']))
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset="Bogus",
processedDataset="Test-Era-v1",
dataTier="DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testMultiCoreReport(self):
"""
_testMultiCoreReport_
Verify that multicore reports can be json encoded and uploaded to couch.
"""
couchdb = CouchServer(os.environ["COUCHURL"])
fwjrdatabase = couchdb.connectDatabase("report_t/fwjrs")
self.mcPath = os.path.join(
getTestBase(), "WMCore_t/FwkJobReport_t/MulticoreReport.pkl")
myReport = Report()
myReport.unpersist(self.mcPath)
fwjrDocument = {
"_id": "303-0",
"jobid": 303,
"retrycount": 0,
"fwjr": myReport.__to_json__(None),
"type": "fwjr"
}
fwjrdatabase.queue(fwjrDocument, timestamp=True)
fwjrdatabase.commit()
return
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(
currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz'
) #file to upload
upfileLog = path.join(
currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt'
) #file to upload
ufc = UserFileCache({
'endpoint': os.environ['UFCURL'],
'pycurl': True
})
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
#hashkey deletion
ufc.removeFile(res['hashkey'])
#log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog + '.downloaded')
self.assertTrue(filecmp.cmp(upfileLog, upfileLog + '.downloaded'))
def testB_ExecuteNonZeroExit(self):
"""
_ExecuteNonZeroExit_
Test the execution of a script
which exits with non-zero code.
"""
self.step.application.command.executable = "brokenCmsRun.py"
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
try:
os.chdir(self.step.builder.workingDir)
executor = StepFactory.getStepExecutor("CMSSW")
executor.initialise(self.step, self.job)
executor.pre()
executor.step.runtime.scramPreScripts.remove("SetupCMSSWPset")
try:
executor.execute()
self.fail("An exception should have been raised")
except WMExecutionFailure as ex:
executor.diagnostic(ex.code, executor, ExceptionInstance=ex)
self.assertEqual(8001, executor.report.getExitCode())
report = Report()
report.load("Report.pkl")
self.assertEqual(8001, report.getExitCode())
except Exception as ex:
self.fail("Failure encountered, %s" % str(ex))
finally:
os.chdir(self.oldCwd)
return
def testC_testViews(self):
"""
_testViews_
Prototype test for what should be a lot of other tests.
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
configCache.document['md5_hash'] = "somemd5"
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.save()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache2.document['md5_hash'] = configCache.document['md5_hash']
configCache2.load()
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache2.delete()
return
def testB_addingConfigsAndTweaks(self):
"""
_addingConfigsAndTweaks_
Test adding config files and tweak files
"""
PSetTweak = "Hello, I am a PSetTweak. It's nice to meet you."
attach = "Hello, I am an attachment"
configCache = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test')
configCache.createUserGroup(groupname="testGroup", username='******')
configCache.setPSetTweaks(PSetTweak=PSetTweak)
configCache.attachments['attach1'] = attach
psetPath = os.path.join(getTestBase(), "WMCore_t/Cache_t/PSet.txt")
configCache.addConfig(newConfig=psetPath, psetHash=None)
configCache.setLabel("sample-label")
configCache.setDescription("describe this config here")
configCache.save()
configString1 = configCache.getConfig()
configCache2 = ConfigCache(os.environ["COUCHURL"],
couchDBName='config_test',
id=configCache.getCouchID(),
rev=configCache.getCouchRev())
configCache2.loadByID(configCache.getCouchID())
configString2 = configCache2.getConfig()
self.assertEqual(configString1, configString2)
self.assertEqual(configCache2.attachments.get('attach1', None), attach)
configCache.delete()
return
def testFNALSiteLocalConfig(self):
"""
_testFNALSiteLocalConfig_
Verify that the FNAL site config file is parsed correctly.
"""
fnalConfigFileName = os.path.join(getTestBase(), "WMCore_t/Storage_t",
"T1_US_FNAL_SiteLocalConfig.xml")
mySiteConfig = SiteLocalConfig(fnalConfigFileName)
assert mySiteConfig.siteName == "T1_US_FNAL", \
"Error: Wrong site name."
assert len(mySiteConfig.eventData.keys()) == 1, \
"Error: Wrong number of event data keys."
assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \
"Eroror: Event data catalog is wrong."
goldenServers = [
"http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier1.cern.ch:8000/FrontierInt",
"http://cmsfrontier2.cern.ch:8000/FrontierInt",
"http://cmsfrontier3.cern.ch:8000/FrontierInt",
"http://cmsfrontier4.cern.ch:8000/FrontierInt"
]
for frontierServer in mySiteConfig.frontierServers:
assert frontierServer in goldenServers, \
"Error: Unknown server: %s" % frontierServer
goldenServers.remove(frontierServer)
assert len(goldenServers) == 0, \
"Error: Missing frontier servers."
goldenProxies = [
"http://cmsfrontier1.fnal.gov:3128",
"http://cmsfrontier2.fnal.gov:3128",
"http://cmsfrontier3.fnal.gov:3128",
"http://cmsfrontier4.fnal.gov:3128"
]
for frontierProxy in mySiteConfig.frontierProxies:
assert frontierProxy in goldenProxies, \
"Error: Unknown proxy: %s" % frontierProxy
goldenProxies.remove(frontierProxy)
assert len(goldenProxies) == 0, \
"Error: Missing proxy servers."
assert mySiteConfig.localStageOut["se-name"] == "cmssrm.fnal.gov", \
"Error: Wrong se name from local stageout."
assert mySiteConfig.localStageOut["command"] == "dccp-fnal", \
"Error: Wrong stage out command."
assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=srmv2", \
"Error: TFC catalog is not correct."
assert mySiteConfig.fallbackStageOut == [], \
"Error: Fallback config is incorrect."
return
def testVanderbiltSiteLocalConfig(self):
"""
_testFNALSiteLocalConfig_
Verify that the FNAL site config file is parsed correctly.
"""
vandyConfigFileName = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T3_US_Vanderbilt_SiteLocalConfig.xml")
mySiteConfig = SiteLocalConfig(vandyConfigFileName)
assert mySiteConfig.siteName == "T3_US_Vanderbilt", \
"Error: Wrong site name."
assert len(mySiteConfig.eventData.keys()) == 1, \
"Error: Wrong number of event data keys."
assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=direct", \
"Eroror: Event data catalog is wrong."
goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier1.cern.ch:8000/FrontierInt",
"http://cmsfrontier2.cern.ch:8000/FrontierInt",
"http://cmsfrontier3.cern.ch:8000/FrontierInt"]
for frontierServer in mySiteConfig.frontierServers:
assert frontierServer in goldenServers, \
"Error: Unknown server: %s" % frontierServer
goldenServers.remove(frontierServer)
assert len(goldenServers) == 0, \
"Error: Missing frontier servers."
goldenProxies = ["http://se1.accre.vanderbilt.edu:3128"]
for frontierProxy in mySiteConfig.frontierProxies:
assert frontierProxy in goldenProxies, \
"Error: Unknown proxy: %s" % frontierProxy
goldenProxies.remove(frontierProxy)
assert len(goldenProxies) == 0, \
"Error: Missing proxy servers."
assert mySiteConfig.localStageOut["se-name"] == "se1.accre.vanderbilt.edu", \
"Error: Wrong se name from local stageout."
assert mySiteConfig.localStageOut["command"] == "srmv2", \
"Error: Wrong stage out command."
assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=srmv2", \
"Error: TFC catalog is not correct."
assert len(mySiteConfig.fallbackStageOut) == 1, \
"Error: Incorrect number of fallback stageout methods"
assert mySiteConfig.fallbackStageOut[0]["command"] == "srmv2-lcg", \
"Error: Incorrect fallback command."
assert mySiteConfig.fallbackStageOut[0]["se-name"] == "se1.accre.vanderbilt.edu", \
"Error: Incorrect fallback SE."
assert mySiteConfig.fallbackStageOut[0]["lfn-prefix"] == "srm://se1.accre.vanderbilt.edu:6288/srm/v2/server?SFN=", \
"Error: Incorrect fallback LFN prefix."
return
def testVanderbiltSiteLocalConfig(self):
"""
_testFNALSiteLocalConfig_
Verify that the FNAL site config file is parsed correctly.
"""
vandyConfigFileName = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T3_US_Vanderbilt_SiteLocalConfig.xml")
mySiteConfig = SiteLocalConfig(vandyConfigFileName)
assert mySiteConfig.siteName == "T3_US_Vanderbilt", \
"Error: Wrong site name."
assert len(mySiteConfig.eventData.keys()) == 1, \
"Error: Wrong number of event data keys."
assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=direct", \
"Eroror: Event data catalog is wrong."
goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier1.cern.ch:8000/FrontierInt",
"http://cmsfrontier2.cern.ch:8000/FrontierInt",
"http://cmsfrontier3.cern.ch:8000/FrontierInt"]
for frontierServer in mySiteConfig.frontierServers:
assert frontierServer in goldenServers, \
"Error: Unknown server: %s" % frontierServer
goldenServers.remove(frontierServer)
assert len(goldenServers) == 0, \
"Error: Missing frontier servers."
goldenProxies = ["http://se1.accre.vanderbilt.edu:3128"]
for frontierProxy in mySiteConfig.frontierProxies:
assert frontierProxy in goldenProxies, \
"Error: Unknown proxy: %s" % frontierProxy
goldenProxies.remove(frontierProxy)
assert len(goldenProxies) == 0, \
"Error: Missing proxy servers."
assert mySiteConfig.localStageOut["se-name"] == "se1.accre.vanderbilt.edu", \
"Error: Wrong se name from local stageout."
assert mySiteConfig.localStageOut["command"] == "srmv2", \
"Error: Wrong stage out command."
assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file://gpfs1/grid/grid-app/cmssoft/cms/SITECONF/local/PhEDEx/storage.xml?protocol=srmv2", \
"Error: TFC catalog is not correct."
assert len(mySiteConfig.fallbackStageOut) == 1, \
"Error: Incorrect number of fallback stageout methods"
assert mySiteConfig.fallbackStageOut[0]["command"] == "srmv2-lcg", \
"Error: Incorrect fallback command."
assert mySiteConfig.fallbackStageOut[0]["se-name"] == "se1.accre.vanderbilt.edu", \
"Error: Incorrect fallback SE."
assert mySiteConfig.fallbackStageOut[0]["lfn-prefix"] == "srm://se1.accre.vanderbilt.edu:6288/srm/v2/server?SFN=", \
"Error: Incorrect fallback LFN prefix."
return
def setUp(self):
"""
_setUp_
Figure out the location of the XML report produced by CMSSW.
"""
self.testInit = TestInitCouchApp(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setupCouch("report_t/fwjrs", "FWJRDump")
self.xmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml")
self.badxmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWFailReport2.xml")
self.skippedFilesxmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedNonExistentFile.xml")
self.skippedAllFilesxmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWSkippedAll.xml")
self.fallbackXmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWInputFallback.xml")
self.twoFileFallbackXmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml")
self.pileupXmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWPileup.xml")
self.testDir = self.testInit.generateWorkDir()
return
def testMultipleInputs(self):
"""
_testMultipleInputs_
Verify that parsing XML reports with multiple inputs works correctly.
"""
xmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMultipleInput.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
assert hasattr(myReport.data.cmsRun1.input, "source"), \
"Error: Report missing input source."
inputFiles = myReport.getInputFilesFromStep("cmsRun1")
assert len(inputFiles) == 2, \
"Error: Wrong number of input files."
for inputFile in inputFiles:
assert inputFile["input_type"] == "primaryFiles", \
"Error: Wrong input type."
assert inputFile["module_label"] == "source", \
"Error: Module label is wrong"
assert inputFile["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \
"Error: Catalog is wrong."
assert inputFile["events"] == 2, \
"Error: Wrong number of events."
assert inputFile["input_source_class"] == "PoolSource", \
"Error: Wrong input source class."
if inputFile["guid"] == "F0875ECD-3347-DF11-9FE0-003048678A80":
assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \
"Error: Input LFN is wrong."
assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \
"Error: Input PFN is wrong."
assert len(inputFile["runs"]) == 1, \
"Error: Wrong number of runs."
assert list(inputFile["runs"])[0].run == 124216, \
"Error: Wrong run number."
assert 1 in list(inputFile["runs"])[0], \
"Error: Wrong lumi sections in input file."
else:
assert inputFile["guid"] == "626D74CE-3347-DF11-9363-0030486790C0", \
"Error: Wrong guid."
assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \
"Error: Input LFN is wrong."
assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \
"Error: Input PFN is wrong."
assert len(inputFile["runs"]) == 1, \
"Error: Wrong number of runs."
assert list(inputFile["runs"])[0].run == 124216, \
"Error: Wrong run number."
assert 2 in list(inputFile["runs"])[0], \
"Error: Wrong lumi sections in input file."
return
def testMultipleInputs(self):
"""
_testMultipleInputs_
Verify that parsing XML reports with multiple inputs works correctly.
"""
xmlPath = os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWMultipleInput.xml")
myReport = Report("cmsRun1")
myReport.parse(xmlPath)
assert hasattr(myReport.data.cmsRun1.input, "source"), \
"Error: Report missing input source."
inputFiles = myReport.getInputFilesFromStep("cmsRun1")
assert len(inputFiles) == 2, \
"Error: Wrong number of input files."
for inputFile in inputFiles:
assert inputFile["input_type"] == "primaryFiles", \
"Error: Wrong input type."
assert inputFile["module_label"] == "source", \
"Error: Module label is wrong"
assert inputFile["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \
"Error: Catalog is wrong."
assert inputFile["events"] == 2, \
"Error: Wrong number of events."
assert inputFile["input_source_class"] == "PoolSource", \
"Error: Wrong input source class."
if inputFile["guid"] == "F0875ECD-3347-DF11-9FE0-003048678A80":
assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \
"Error: Input LFN is wrong."
assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/F0875ECD-3347-DF11-9FE0-003048678A80.root", \
"Error: Input PFN is wrong."
assert len(inputFile["runs"]) == 1, \
"Error: Wrong number of runs."
assert list(inputFile["runs"])[0].run == 124216, \
"Error: Wrong run number."
assert 1 in list(inputFile["runs"])[0], \
"Error: Wrong lumi sections in input file."
else:
assert inputFile["guid"] == "626D74CE-3347-DF11-9363-0030486790C0", \
"Error: Wrong guid."
assert inputFile["lfn"] == "/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \
"Error: Input LFN is wrong."
assert inputFile["pfn"] == "dcap://cmsdca3.fnal.gov:24142/pnfs/fnal.gov/usr/cms/WAX/11/store/backfill/2/unmerged/WMAgentCommissioining10/MinimumBias/RECO/rereco_GR10_P_V4_All_v1/0000/626D74CE-3347-DF11-9363-0030486790C0.root", \
"Error: Input PFN is wrong."
assert len(inputFile["runs"]) == 1, \
"Error: Wrong number of runs."
assert list(inputFile["runs"])[0].run == 124216, \
"Error: Wrong run number."
assert 2 in list(inputFile["runs"])[0], \
"Error: Wrong lumi sections in input file."
return
def setUp(self):
"""
_setUp_
Setup a dashboard reporter
"""
self.reporter = DashboardReporter(config=None)
self.processingReport = ProcessingSample.report
self.mergeReport = MergeSample.report
self.errorReport = ErrorSample.report
self.fallbackReport = FallbackSample.report
self.twoFileFallbackXmlPath = os.path.join(
getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWTwoFileRemote.xml")
self.pileupXmlPath = os.path.join(
getTestBase(), "WMCore_t/FwkJobReport_t/CMSSWPileup.xml")
return
def testCreateStageOutCommand_realFile(self):
base = os.path.join(getTestBase(),
"WMCore_t/Storage_t/ExecutableCommands.py")
result = self.CPImpl.createStageOutCommand(base, "test")
expectedResult = [
" '590' -eq $DEST_SIZE", "DEST_SIZE=`/bin/ls -l test", base
]
for text in expectedResult:
self.assertIn(text, result)
def testFNALSiteLocalConfig(self):
"""
_testFNALSiteLocalConfig_
Verify that the FNAL site config file is parsed correctly.
"""
fnalConfigFileName = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_US_FNAL_SiteLocalConfig.xml")
mySiteConfig = SiteLocalConfig(fnalConfigFileName)
assert mySiteConfig.siteName == "T1_US_FNAL", \
"Error: Wrong site name."
assert len(mySiteConfig.eventData.keys()) == 1, \
"Error: Wrong number of event data keys."
assert mySiteConfig.eventData["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=dcap", \
"Eroror: Event data catalog is wrong."
goldenServers = ["http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier.cern.ch:8000/FrontierInt",
"http://cmsfrontier1.cern.ch:8000/FrontierInt",
"http://cmsfrontier2.cern.ch:8000/FrontierInt",
"http://cmsfrontier3.cern.ch:8000/FrontierInt",
"http://cmsfrontier4.cern.ch:8000/FrontierInt"]
for frontierServer in mySiteConfig.frontierServers:
assert frontierServer in goldenServers, \
"Error: Unknown server: %s" % frontierServer
goldenServers.remove(frontierServer)
assert len(goldenServers) == 0, \
"Error: Missing frontier servers."
goldenProxies = ["http://cmsfrontier1.fnal.gov:3128",
"http://cmsfrontier2.fnal.gov:3128",
"http://cmsfrontier3.fnal.gov:3128",
"http://cmsfrontier4.fnal.gov:3128"]
for frontierProxy in mySiteConfig.frontierProxies:
assert frontierProxy in goldenProxies, \
"Error: Unknown proxy: %s" % frontierProxy
goldenProxies.remove(frontierProxy)
assert len(goldenProxies) == 0, \
"Error: Missing proxy servers."
assert mySiteConfig.localStageOut["se-name"] == "cmssrm.fnal.gov", \
"Error: Wrong se name from local stageout."
assert mySiteConfig.localStageOut["command"] == "dccp-fnal", \
"Error: Wrong stage out command."
assert mySiteConfig.localStageOut["catalog"] == "trivialcatalog_file:/uscmst1/prod/sw/cms/SITECONF/T1_US_FNAL/PhEDEx/storage.xml?protocol=srmv2", \
"Error: TFC catalog is not correct."
assert mySiteConfig.fallbackStageOut == [], \
"Error: Fallback config is incorrect."
return
def testMultipleRegexMatch(self):
# Check that an lfn is converted to the right pfn
in_lfn = '/store/user/fred/data'
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_CH_CERNBOX_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Florida_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_ES_IFCA_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Nebraska_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
def testMultipleRegexMatch(self):
# Check that an lfn is converted to the right pfn
in_lfn = '/store/user/fred/data'
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_CH_CERNBOX_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "root://eosuser.cern.ch/eos/user/f/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_PT_NCG_Lisbon_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ncg.ingrid.pt:8444/srm/managerv2?SFN=/cmst3/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Florida_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm.ihepa.ufl.edu:8443/srm/v2/server?SFN=/cms/data/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_ES_IFCA_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://srm01.ifca.es:8444/srm/managerv2?SFN=/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
tfc_file = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T2_US_Nebraska_TrivialFileCatalog.xml")
tfc = readTFC(tfc_file)
out_pfn = "srm://dcache07.unl.edu:8443/srm/v2/server?SFN=/mnt/hadoop/user/uscms01/pnfs/unl.edu/data4/cms/store/user/fred/data"
pfn = tfc.matchLFN('srmv2', in_lfn)
self.assertEqual(out_pfn, pfn)
def testExceptionFilter(self):
"""
Test getWMExceptionStr function.
"""
count = 0
logPath = os.path.join(getTestBase(), "WMCore_t/test_condor.log")
for result in listWMExceptionStr(logPath):
# print(result)
count += 1
self.assertEqual(count, 4)
def setupJobEnvironment(self, name = 'test'):
"""
_setupJobEnvironment_
Make some sort of environment in which to run tests
"""
os.environ['WMAGENT_SITE_CONFIG_OVERRIDE'] = os.path.join(getTestBase(),
"WMCore_t/Storage_t",
"T1_US_FNAL_SiteLocalConfig.xml")
return
def setUp(self):
filePath = os.path.join(getTestBase(),
"WMCore_t/WorkQueue_t/DataStructs_t/wq_available_elements.json")
with open(filePath, "r") as f:
gqData = JsonWrapper.load(f)
self.gqElements = []
for ele in gqData:
self.gqElements.append(WorkQueueElement(**ele))
def testChecksum(self):
"""
Tests checksum method
"""
checksum1 = calculateChecksum(
tarfile_=path.join(
getTestBase(), "WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_111229_140959_publish.tgz"
)
)
checksum2 = calculateChecksum(
tarfile_=path.join(
getTestBase(),
"WMCore_t/Services_t/UserFileCache_t/ewv_crab_EwvAnalysis_31_resubmit_111229_144319_publish.tgz",
)
)
self.assertTrue(checksum1)
self.assertTrue(checksum2)
self.assertFalse(checksum1 == checksum2)
self.assertRaises(IOError, calculateChecksum, **{"tarfile_": "does_not_exist"})
return
def testUploadDownload(self):
if "UFCURL" in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload
ufc = UserFileCache({"endpoint": os.environ["UFCURL"]})
# named upload/download
res = ufc.upload(upfile, "name_publish.tgz")
ufc.download(name=res["name"], output="name_publish.tgz")
# hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload
ufc = UserFileCache({'endpoint':os.environ['UFCURL']})
#named upload/download
res = ufc.upload(upfile, 'name_publish.tgz')
ufc.download(name=res['name'], output='name_publish.tgz')
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
