def testLen(self):
"""
__testLen__
Test that the __len__ function will actualy return the correct length.
"""
#This is simple. It should just have a length equal to the number of committed
#And yet to be committed jobs
testJobA = Job()
testJobB = Job()
testJobC = Job()
testJobD = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
self.assertEqual(len(testJobGroup), 2)
testJobGroup.add(testJobC)
self.assertEqual(len(testJobGroup), 3)
testJobGroup.commit()
testJobGroup.add(testJobD)
self.assertEqual(len(testJobGroup), 4)
return
def testFirstEventMC(self):
"""
_testFirstEventMC_
Verify that we set the firstEvent parameter whenever the FirstEvent
field in the job mask is a positive integer and the job is a production
one. Otherwise we get a number based on the counter (modulo 2^32 - 1)
"""
job = Job()
job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
job["mask"] = Mask()
job["counter"] = 5
job["mask"]["FirstLumi"] = 200
try:
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertRaises(WMTweakMaskError, WMTweaks.makeJobTweak, job)
except WMTweakMaskError:
pass
job["mask"]["FirstEvent"] = 100
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertFalse(
hasattr(tweak.process.source, "skipEvents"),
"Error: There should be no skipEvents tweak, it's MC.")
self.assertTrue(hasattr(tweak.process.source, "firstEvent"),
"Error: There should be a first event tweak")
self.assertEqual(tweak.process.source.firstEvent,
'customTypeCms.untracked.uint32(100)',
"Error: The firstEvent tweak should be 100.")
return
def testMask3(self):
mymask = Mask()
mymask['FirstEvent'] = 9999
mymask['LastEvent'] = 999
myjob = Job()
myjob["mask"] = mymask
self.roundTrip(myjob)
def testAddCommit(self):
"""
_testAddCommit_
Test the add() and commit() methods of the JobGroup class. Verify that
jobs are not returned from getJobs() until commit() has been called.
"""
testJob = Job()
testJobGroup = JobGroup()
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs before jobs have been added."
testJobGroup.add(testJob)
assert len(testJobGroup.getJobs()) == 0, \
"ERROR: JobGroup has jobs commit() was called."
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 1, \
"ERROR: JobGroup has wrong number of jobs."
assert testJob in testJobGroup.getJobs(), \
"ERROR: JobGroup has unknown jobs."
return
def testFirstLumiMC(self):
"""
_testFirstLumiMC_
Verify that we set the lumi in a MC job and it gets into
process.source.firstRun parameter, and if we don't at least we
get the counter there.
"""
job = Job()
job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
job["mask"] = Mask()
job["counter"] = 5
job["mask"]["FirstEvent"] = 100
try:
tweak = WMTweaks.makeJobTweak(job)
self.assertRaises(WMTweakMaskError, WMTweaks.makeJobTweak, job)
except WMTweakMaskError:
pass
job["mask"]["FirstLumi"] = 200
tweak = WMTweaks.makeJobTweak(job)
self.assertTrue(hasattr(tweak.process.source, "firstLuminosityBlock"),
"Error: There should be a first lumi tweak")
self.assertEqual(tweak.process.source.firstLuminosityBlock, 200,
"Error: The first luminosity block should be 5")
job["mask"]["FirstLumi"] = 10
tweak = WMTweaks.makeJobTweak(job)
self.assertTrue(hasattr(tweak.process.source, "firstLuminosityBlock"),
"Error: There should be a first lumi tweak")
self.assertEqual(tweak.process.source.firstLuminosityBlock, 10,
"Error: The first luminosity block should be 10")
def testFirstEvent(self):
"""
_testFirstEvent_
Verify that we set the firstEvent parameter whenever the FirstEvent
field in the job mask is a positive integer. And the job is
not production.
"""
job = Job()
job["input_files"] = [{"lfn": "bogusFile", "parents": []}]
job["mask"] = Mask()
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertFalse(hasattr(tweak.process.source, "skipEvents"),
"Error: There should be no skipEvents tweak.")
self.assertFalse(hasattr(tweak.process.source, "firstEvent"),
"Error: There should be no firstEvent tweak.")
job["mask"]["FirstEvent"] = 0
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "skipEvents"),
"Error: There should be a skipEvents tweak.")
self.assertEqual(tweak.process.source.skipEvents,
'customTypeCms.untracked.uint32(0)',
"Error: The skipEvents tweak should be 0.")
return
def setUp(self):
"""
_setUp_
Initial Setup for the Job Testcase
"""
self.inputFiles = []
for i in range(1, 1000):
lfn = "/store/data/%s/%s/file.root" % (random.randint(
1000, 9999), random.randint(1000, 9999))
size = random.randint(1000, 2000)
events = 1000
run = random.randint(0, 2000)
lumi = random.randint(0, 8)
file = File(lfn=lfn,
size=size,
events=events,
checksums={"cksum": "1"})
file.addRun(Run(run, *[lumi]))
self.inputFiles.append(file)
self.dummyJob = Job(files=self.inputFiles)
return
def setUp(self):
# stolen from CMSSWExecutor_t. thanks, dave
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
stepHelper = step = self.task.makeStep("DeleteTest")
self.step = stepHelper.data
self.actualStep = stepHelper
template = DeleteTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.executor = StepFactory.getStepExecutor(self.actualStep.stepType())
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.insert(0, self.testDir)
sys.path.insert(0, self.sandboxDir)
self.job = Job(name = "/UnitTest/DeleterTask/DeleteTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testFirstRunMC(self):
"""
_testFirstRunMC_
Verify that we set the lumi in a MC job and it gets into
process.source.firstRun parameter.
"""
job = Job()
job["input_files"] = [{"lfn": "MCFakeFile", "parents": []}]
job["mask"] = Mask()
job["mask"]["FirstLumi"] = 200
job["mask"]["FirstEvent"] = 100
job["counter"] = 5
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "firstRun"),
"Error: There should be a first run tweak")
self.assertEqual(tweak.process.source.firstRun,
'customTypeCms.untracked.uint32(1)',
"Error: The first run should be 1")
job["mask"]["FirstRun"] = 5
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "firstRun"),
"Error: There should be a first run tweak")
self.assertEqual(tweak.process.source.firstRun,
'customTypeCms.untracked.uint32(5)',
"Error: The first run should be 5")
def testA(self):
"""instantiate"""
document = Document()
document[u'pset_tweak_details'] = {}
document[u'pset_tweak_details'][u'process'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed1'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed2'] = {}
document[u'pset_tweak_details'][u'process'][
u'RandomNumberGeneratorService'][u'seed3'] = {}
document = self.database.commitOne(document)[0]
seeder = ReproducibleSeeding(CouchUrl=self.testInit.couchUrl,
CouchDBName=self.testInit.couchDbName,
ConfigCacheDoc=document[u'id'])
job = Job("testjob")
seeder(job)
baggage = job.getBaggage()
seed1 = getattr(baggage.process.RandomNumberGeneratorService, "seed1",
None)
self.assertTrue(seed1 != None)
def testPersist(self):
"""
_testPersist_
Verify that we're able to save and load the job package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def testAddingJobs(self):
"""
_testAddingJobs_
Verify that adding jobs to the package works as expected.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
package[i] = newJob
# There is an extra key for the directory the package is stored in.
assert len(package.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = package[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
return
def setUp(self):
"""
_setUp_
Build a testing environment similar to a WN
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# Build a workload/task/step with the basic required information
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
self.step.output.jobReport = "FrameworkJobReport.xml"
self.helper.addOutputModule("outputRECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.addOutputModule("outputALCARECORECO", primaryDataset = "Bogus",
processedDataset = "Test-Era-v1",
dataTier = "DATA")
self.helper.setGlobalTag("Bogus")
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
# Build the TaskSpace/StepSpace
self.sandboxDir = os.path.join(self.testDir, "UnitTests")
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
# Copy the files that cmsRun would have generated in the step space
open(os.path.join(self.step.builder.workingDir, "outputRECORECO.root"), "w").close()
open(os.path.join(self.step.builder.workingDir, "outputALCARECORECO.root"), "w").close()
shutil.copy(os.path.join(getTestBase(),
"WMCore_t/FwkJobReport_t/CMSSWProcessingReport.xml"),
os.path.join(self.step.builder.workingDir, "FrameworkJobReport.xml"))
# Create a job
self.job = Job(name = "/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
self.job["id"] = 1
# Set the PATH
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
self.oldCwd = os.getcwd()
def testFirstRun(self):
"""
_testFirstRun_
Verify that when we set the FirstRun in the mask, it is set in the
process but when it is not, then no firstRun appears in the process.
This for jobs with real input files.
"""
job = Job()
job["input_files"] = [{"lfn": "bogusFile", "parents": []}]
job["mask"] = Mask()
tweak = PSetTweak()
WMTweaks.makeJobTweak(job, tweak)
self.assertFalse(hasattr(tweak.process.source, "firstRun"),
"Error: There should be no firstRun tweak.")
job["mask"]["FirstRun"] = 93
tweak = WMTweaks.makeJobTweak(job, tweak)
self.assertTrue(hasattr(tweak.process.source, "firstRun"),
"Error: There should be a firstRun tweak.")
self.assertEqual(tweak.process.source.firstRun,
'customTypeCms.untracked.uint32(93)',
"Error: The firstRun tweak should be 93.")
return
def testCreate(self):
"""
_testCreate_
Test the JobGroup constructor and passing different job containers
into it.
"""
testSubscription = Subscription()
testJobGroupA = JobGroup(subscription = testSubscription)
assert testJobGroupA.subscription == testSubscription, \
"ERROR: Failed to pass subscription in constructor"
assert len(testJobGroupA.jobs) == 0 and len(testJobGroupA.newjobs) == 0, \
"ERROR: JobGroup not empty on creation"
testJobA = Job()
testJobB = Job()
testJobGroupB = JobGroup(jobs = [testJobA, testJobB])
assert testJobGroupB.jobs == [], \
"ERROR: Jobs committed to jobgroup too soon."
jobGroupJobs = testJobGroupB.newjobs
goldenJobs = [testJobA, testJobB]
for job in jobGroupJobs:
assert job in goldenJobs, \
"ERROR: Extra job in job group"
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Job missing from job group"
testJobGroupC = JobGroup(jobs = testJobA)
assert testJobGroupC.jobs == [], \
"ERROR: Jobs committed to jobgroup too soon."
jobGroupJobs = testJobGroupC.newjobs
assert len(jobGroupJobs) == 1, \
"ERROR: Wrong number of jobs in jobgroup."
assert testJobA in jobGroupJobs, \
"ERROR: Wrong job in jobgroup."
return
def testBaggage(self):
"""
_testBaggage_
Verify that job baggage is persisted with the package.
"""
package = JobPackage()
for i in range(100):
newJob = Job("Job%s" % i)
newJob["id"] = i
baggage = newJob.getBaggage()
setattr(baggage, "thisJob", newJob["name"])
setattr(baggage, "seed1", 11111111)
setattr(baggage, "seed2", 22222222)
setattr(baggage, "seed3", 33333333)
setattr(baggage, "seed4", 44444444)
setattr(baggage, "seed5", 55555555)
package[i] = newJob
package.save(self.persistFile)
assert os.path.exists(self.persistFile), \
"Error: Package file was never created."
newPackage = JobPackage()
newPackage.load(self.persistFile)
# There is an extra key for the directory the package is stored in.
assert len(newPackage.keys()) == 101, \
"Error: Wrong number of jobs in package."
for i in range(100):
job = newPackage[i]
assert job["id"] == i, \
"Error: Jobs has wrong ID."
assert job["name"] == "Job%d" % i, \
"Error: Job has wrong name."
jobBaggage = job.getBaggage()
assert jobBaggage.thisJob == "Job%d" % i, \
"Error: Job baggage has wrong name."
assert jobBaggage.seed1 == 11111111, \
"Error: Job baggee has wrong value for seed1."
assert jobBaggage.seed2 == 22222222, \
"Error: Job baggee has wrong value for seed2."
assert jobBaggage.seed3 == 33333333, \
"Error: Job baggee has wrong value for seed3."
assert jobBaggage.seed4 == 44444444, \
"Error: Job baggee has wrong value for seed4."
assert jobBaggage.seed5 == 55555555, \
"Error: Job baggee has wrong value for seed5."
return
def createTestJob(self, fwjr):
"""
_createTestJob_
Creates a minimal job to report
"""
job = Job('finishedJob')
job['retry_count'] = 1
job['workflow'] = 'testWorkflow'
job['fwjr'] = fwjr
return job
def createTestJob(self):
"""
_createTestJob_
Create a test job that has parents for each input file.
"""
newJob = Job(name = "TestJob")
newJob.addFile(File(lfn = "/some/file/one",
parents = set([File(lfn = "/some/parent/one")])))
newJob.addFile(File(lfn = "/some/file/two",
parents = set([File(lfn = "/some/parent/two")])))
return newJob
def testGetJobs(self):
"""
_testGetJobs_
Verify that the getJobs() method of the JobGroup class returns the
correct output for each output container type it supports.
"""
testJobA = Job()
testJobB = Job()
testJobGroup = JobGroup(jobs = [testJobA, testJobB])
testJobGroup.commit()
assert len(testJobGroup.getJobs()) == 2, \
"ERROR: Wrong number of jobs in job group"
goldenJobs = [testJobA, testJobB]
for job in testJobGroup.getJobs():
assert job in goldenJobs, \
"ERROR: Unknown Job in JobGroup."
goldenJobs.remove(job)
assert len(goldenJobs) == 0, \
"ERROR: Jobs are missing from the JobGroup."
goldenIDs = []
goldenIDs.append(testJobA["id"])
goldenIDs.append(testJobB["id"])
for jobID in testJobGroup.getJobs(type = "id"):
assert jobID in goldenIDs, \
"ERROR: Unknown JobID in JobGroup"
goldenIDs.remove(jobID)
assert len(goldenIDs) == 0, \
"ERROR: Job IDs are missing from the JobGroup."
return
def testC(self):
"""test building a tweak from the seeds"""
job = Job("TestJob")
seeder = AutomaticSeeding()
job.addBaggageParameter("process.RandomNumberGeneratorService.seed1.initialSeed", 123445)
job.addBaggageParameter("process.RandomNumberGeneratorService.seed2.initialSeed", 123445)
job.addBaggageParameter("process.RandomNumberGeneratorService.seed3.initialSeed", 7464738)
job.addBaggageParameter("process.RandomNumberGeneratorService.seed44.initialSeed", 98273762)
seeder(job)
tweak = PSetTweak()
for x in job.baggage.process.RandomNumberGeneratorService:
parameter = "process.RandomNumberGeneratorService.%s.initialSeed" % x._internal_name
tweak.addParameter(parameter, x.initialSeed)
print(tweak)
def createTestJob(self):
"""
Create a test job to pass to the DashboardInterface
"""
job = Job(name = "ThisIsASillyName")
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(1, *[45]))
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(1, *[46]))
job.addFile(testFileA)
job.addFile(testFileB)
job['id'] = 1
return job
def createJobs(totalJobs=100):
"""
_createJobs_
"""
global lastJobID
newJobs = []
for i in range(totalJobs):
newJob = Job(name=makeUUID(), files=[createFile(), createFile()])
newJob["task"] = "SomeTask"
newJob["workflow"] = "SomeWorkflow"
newJob["owner"] = "*****@*****.**"
newJob["id"] = lastJobID
newJob["couch_record"] = None
newJobs.append(newJob)
lastJobID += 1
return newJobs
def setUp(self):
"""
build a step for testing purposes
"""
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("CMSSWExecutor")
stepHelper = step = self.task.makeStep("ExecutorTest")
self.step = stepHelper.data
template = CMSSWTemplate()
template(self.step)
self.helper = template.helper(self.step)
self.step.application.setup.scramCommand = "scramulator.py"
self.step.application.command.executable = "cmsRun.py"
self.step.application.setup.scramProject = "CMSSW"
self.step.application.setup.scramArch = "slc5_ia32_gcc434"
self.step.application.setup.cmsswVersion = "CMSSW_X_Y_Z"
self.step.application.setup.softwareEnvironment = "echo \"Software Setup...\";"
taskMaker = TaskMaker(self.workload, self.testDir)
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.sandboxDir = "%s/UnitTests" % self.testDir
self.task.build(self.testDir)
sys.path.append(self.testDir)
sys.path.append(self.sandboxDir)
self.job = Job(name="/UnitTest/CMSSWExecutor/ExecutorTest-test-job")
binDir = inspect.getsourcefile(ModuleLocator)
binDir = binDir.replace("__init__.py", "bin")
if not binDir in os.environ['PATH']:
os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
def testB(self):
"""test plugin acts on a Job as expected"""
job = Job("TestJob")
seeder = AutomaticSeeding()
seeder(job)
def testProcessing(self):
"""
_testProcessing_
Setup a processing workflow and job and verify that the FWJR produced
by the emulator is reasonable.
"""
rerecoTask = self.workload.getTask("DataProcessing")
cmsRunStep = rerecoTask.getStep("cmsRun1")
inputFile = File(lfn="/path/to/test/lfn",
size=1048576,
events=1000,
merged=True)
inputFile.addRun(Run(1, *[1, 2, 3, 4, 5]))
inputFile.addRun(Run(2, *[1, 2, 3, 4, 5, 6]))
processingJob = Job(name="ProcessingJob", files=[inputFile])
processingJob["task"] = "/Tier1ReReco/ReReco"
processingJob["mask"].setMaxAndSkipEvents(500, 0)
processingJob["id"] = 1
processingJob["location"] = "cmssrm.fnal.gov"
emu = ReportEmu(WMStep=cmsRunStep.getTypeHelper(), Job=processingJob)
report = emu()
reportInputFiles = report.getInputFilesFromStep("cmsRun1")
assert len(reportInputFiles) == 1, \
"Error: Wrong number of input files for the job."
assert reportInputFiles[0]["lfn"] == inputFile["lfn"], \
"Error: Input LFNs do not match: %s" % reportInputFiles[0]["lfn"]
assert reportInputFiles[0]["size"] == inputFile["size"], \
"Error: Input file sizes do not match."
assert reportInputFiles[0]["events"] == inputFile["events"], \
"Error: Input file events do not match."
goldenRuns = [Run(1, *[1, 2, 3, 4, 5]), Run(2, *[1, 2, 3, 4, 5, 6])]
assert len(reportInputFiles[0]["runs"]) == len(goldenRuns), \
"Error: Wrong number of runs in input file."
for inputRun in reportInputFiles[0]["runs"]:
for goldenRun in goldenRuns:
if inputRun.run == goldenRun.run:
goldenRun.lumis.sort()
inputRun.lumis.sort()
if goldenRun.lumis == inputRun.lumis:
goldenRuns.remove(goldenRun)
break
assert len(goldenRuns) == 0, \
"Error: Run information wrong on input file."
recoOutputFiles = report.getFilesFromOutputModule(
"cmsRun1", "RECOoutput")
alcaOutputFiles = report.getFilesFromOutputModule(
"cmsRun1", "ALCARECOoutput")
assert len(recoOutputFiles) == 1, \
"Error: There should only be one RECO output file."
assert len(alcaOutputFiles) == 1, \
"Error: There should only be one ALCA output file."
assert recoOutputFiles[0]["module_label"] == "RECOoutput", \
"Error: RECO file has wrong output module."
assert alcaOutputFiles[0]["module_label"] == "ALCARECOoutput", \
"Error: ALCA file has wrong output module."
self.verifyOutputMetaData(recoOutputFiles[0], processingJob)
self.verifyOutputMetaData(alcaOutputFiles[0], processingJob)
dataTierMap = {"RECOoutput": "RECO", "ALCARECOoutput": "ALCARECO"}
for outputFile in [recoOutputFiles[0], alcaOutputFiles[0]]:
assert outputFile["dataset"]["applicationName"] == "cmsRun", \
"Error: Application name is incorrect."
assert outputFile["dataset"]["primaryDataset"] == self.primaryDataset, \
"Error: Primary dataset is incorrect."
assert outputFile["dataset"]["dataTier"] == dataTierMap[outputFile["module_label"]], \
"Error: Data tier is incorrect."
return
def setUp(self):
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
# shut up SiteLocalConfig
os.environ['CMS_PATH'] = os.getcwd()
workload = copy.deepcopy(testWorkloads.workload)
task = workload.getTask("Production")
step = task.getStep("stageOut1")
# want to get the cmsstep so I can make the Report
cmsstep = task.getStep('cmsRun1')
self.cmsstepdir = os.path.join(self.testDir, 'cmsRun1')
os.mkdir(self.cmsstepdir)
open(os.path.join(self.cmsstepdir, '__init__.py'), 'w').close()
open(os.path.join(self.cmsstepdir, 'Report.pkl'), 'w').close()
cmsbuilder = CMSSWBuilder.CMSSW()
cmsbuilder(cmsstep.data, 'Production', self.cmsstepdir)
realstep = StageOutTemplate.StageOutStepHelper(step.data)
realstep.disableRetries()
self.realstep = realstep
self.stepDir = os.path.join(self.testDir, 'stepdir')
os.mkdir(self.stepDir)
builder = StageOutBuilder.StageOut()
builder(step.data, 'Production', self.stepDir)
# stolen from CMSSWExecutor_t. thanks, dave
# first, delete all the sandboxen and taskspaces
# because of caching, this leaks from other tests in other files
# this sucks because the other tests are using sandboxen that
# are deleted after the test is over, which causes theses tests
# to break
modsToDelete = []
# not sure what happens if you delete from
# an arrey you're iterating over. doing it in
# two steps
for modname in sys.modules.keys():
# need to blow away things in sys.modules, otherwise
# they are cached and we look at old taskspaces
if modname.startswith('WMTaskSpace'):
modsToDelete.append(modname)
if modname.startswith('WMSandbox'):
modsToDelete.append(modname)
for modname in modsToDelete:
try:
reload(sys.modules[modname])
except Exception:
pass
del sys.modules[modname]
self.oldpath = sys.path[:]
self.testInit = TestInit(__file__)
self.testDir = self.testInit.generateWorkDir()
self.job = Job(name="/UnitTests/DeleterTask/DeleteTest-test-job")
shutil.copyfile('/etc/hosts', os.path.join(self.testDir, 'testfile'))
self.workload = newWorkload("UnitTests")
self.task = self.workload.newTask("DeleterTask")
cmsswHelper = self.task.makeStep("cmsRun1")
cmsswHelper.setStepType('CMSSW')
stepHelper = cmsswHelper.addStep("DeleteTest")
stepHelper.setStepType('StageOut')
self.cmsswstep = cmsswHelper.data
self.cmsswHelper = cmsswHelper
self.stepdata = stepHelper.data
self.stephelp = StageOutTemplate.StageOutStepHelper(stepHelper.data)
self.task.applyTemplates()
self.executor = StepFactory.getStepExecutor(self.stephelp.stepType())
taskMaker = TaskMaker(self.workload, os.path.join(self.testDir))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
self.task.build(os.path.join(self.testDir, 'UnitTests'))
sys.path.insert(0, self.testDir)
sys.path.insert(0, os.path.join(self.testDir, 'UnitTests'))
# binDir = inspect.getsourcefile(ModuleLocator)
# binDir = binDir.replace("__init__.py", "bin")
#
# if not binDir in os.environ['PATH']:
# os.environ['PATH'] = "%s:%s" % (os.environ['PATH'], binDir)
open(os.path.join(self.testDir, 'UnitTests', '__init__.py'),
'w').close()
shutil.copyfile(
os.path.join(os.path.dirname(__file__), 'MergeSuccess.pkl'),
os.path.join(self.testDir, 'UnitTests', 'WMTaskSpace', 'cmsRun1',
'Report.pkl'))
