def notestTwoJobGroups(self):
"""
Test two job groups with a shared fileset. (Minimal part of testGetLocations which was failing)
"""
testWorkflow1 = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test1")
testWorkflow1.create()
testWMBSFileset1 = WMBSFileset(name="TestFileset1")
testWMBSFileset1.create()
testSubscription1 = Subscription(fileset=testWMBSFileset1,
workflow=testWorkflow1)
testSubscription1.create()
testJobGroup1 = JobGroup(subscription=testSubscription1)
testJobGroup1.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.create()
testJobA = Job(name="TestJobA")
testJobA.addFile(testFileA)
testJobGroup1.add(testJobA)
testJobGroup1.commit()
testWorkflow2 = Workflow(spec="spec.xml",
owner="Simon",
name="wf002",
task="Test2")
testWorkflow2.create()
testWMBSFileset2 = WMBSFileset(name="TestFileset1")
testWMBSFileset2.create()
testSubscription2 = Subscription(fileset=testWMBSFileset2,
workflow=testWorkflow2)
testSubscription2.create()
testJobGroup2 = JobGroup(subscription=testSubscription2)
testJobGroup2.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.create()
testJobA1 = Job(name="TestJobA1")
testJobA1.addFile(testFileC)
testJobGroup2.add(testJobA1)
testJobGroup2.commit()
def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site = None, bl = [], wl = []):
"""
_makeNJobs_
Make and return a WMBS Job and File
This handles all those damn add-ons
"""
# Set the CacheDir
cacheDir = os.path.join(self.testDir, 'CacheDir')
for n in range(nJobs):
# First make a file
#site = self.sites[0]
testFile = File(lfn = "/singleLfn/%s/%s" %(name, n),
size = 1024, events = 10)
if site:
testFile.setLocation(site)
else:
for tmpSite in self.sites:
testFile.setLocation('se.%s' % (tmpSite))
testFile.create()
fileset.addFile(testFile)
fileset.commit()
index = 0
for f in fileset.files:
index += 1
testJob = Job(name = '%s-%i' %(name, index))
testJob.addFile(f)
testJob["location"] = f.getLocations()[0]
testJob['custom']['location'] = f.getLocations()[0]
testJob['task'] = task.getPathName()
testJob['sandbox'] = task.data.input.sandbox
testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl')
testJob['mask']['FirstEvent'] = 101
testJob['owner'] = 'tapas'
testJob["siteBlacklist"] = bl
testJob["siteWhitelist"] = wl
testJob['ownerDN'] = 'tapas'
testJob['ownerRole'] = 'cmsrole'
testJob['ownerGroup'] = 'phgroup'
jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub), 'Job_%i' % (index))
os.makedirs(jobCache)
testJob.create(jobGroup)
testJob['cache_dir'] = jobCache
testJob.save()
jobGroup.add(testJob)
output = open(os.path.join(jobCache, 'job.pkl'),'w')
pickle.dump(testJob, output)
output.close()
return testJob, testFile
def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site):
"""
_makeNJobs_
Make and return a WMBS Job and File
This handles all those damn add-ons
"""
# Set the CacheDir
cacheDir = os.path.join(self.testDir, 'CacheDir')
for n in range(nJobs):
# First make a file
# site = self.sites[0]
testFile = File(lfn="/singleLfn/%s/%s" % (name, n),
size=1024,
events=10)
fileset.addFile(testFile)
fileset.commit()
location = None
if isinstance(site, list):
if len(site) > 0:
location = site[0]
else:
location = site
index = 0
for f in fileset.files:
index += 1
testJob = Job(name='%s-%i' % (name, index))
testJob.addFile(f)
testJob["location"] = location
testJob["possiblePSN"] = set(site) if isinstance(
site, list) else set([site])
testJob['task'] = task.getPathName()
testJob['sandbox'] = task.data.input.sandbox
testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl')
testJob['mask']['FirstEvent'] = 101
testJob['priority'] = 101
testJob['numberOfCores'] = 1
jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub),
'Job_%i' % (index))
os.makedirs(jobCache)
testJob.create(jobGroup)
testJob['cache_dir'] = jobCache
testJob.save()
jobGroup.add(testJob)
output = open(os.path.join(jobCache, 'job.pkl'), 'w')
pickle.dump(testJob, output)
output.close()
return testJob, testFile
def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site=None, bl=[], wl=[]):
"""
_makeNJobs_
Make and return a WMBS Job and File
This handles all those damn add-ons
"""
# Set the CacheDir
cacheDir = os.path.join(self.testDir, "CacheDir")
for n in range(nJobs):
# First make a file
# site = self.sites[0]
testFile = File(lfn="/singleLfn/%s/%s" % (name, n), size=1024, events=10)
if site:
testFile.setLocation(site)
else:
for tmpSite in self.sites:
testFile.setLocation("se.%s" % (tmpSite))
testFile.create()
fileset.addFile(testFile)
fileset.commit()
index = 0
for f in fileset.files:
index += 1
testJob = Job(name="%s-%i" % (name, index))
testJob.addFile(f)
testJob["location"] = f.getLocations()[0]
testJob["custom"]["location"] = f.getLocations()[0]
testJob["task"] = task.getPathName()
testJob["sandbox"] = task.data.input.sandbox
testJob["spec"] = os.path.join(self.testDir, "basicWorkload.pcl")
testJob["mask"]["FirstEvent"] = 101
testJob["owner"] = "tapas"
testJob["siteBlacklist"] = bl
testJob["siteWhitelist"] = wl
testJob["ownerDN"] = "tapas"
testJob["ownerRole"] = "cmsrole"
testJob["ownerGroup"] = "phgroup"
jobCache = os.path.join(cacheDir, "Sub_%i" % (sub), "Job_%i" % (index))
os.makedirs(jobCache)
testJob.create(jobGroup)
testJob["cache_dir"] = jobCache
testJob.save()
jobGroup.add(testJob)
output = open(os.path.join(jobCache, "job.pkl"), "w")
pickle.dump(testJob, output)
output.close()
return testJob, testFile
def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site, bl = [], wl = []):
"""
_makeNJobs_
Make and return a WMBS Job and File
This handles all those damn add-ons
"""
# Set the CacheDir
cacheDir = os.path.join(self.testDir, 'CacheDir')
for n in range(nJobs):
# First make a file
#site = self.sites[0]
testFile = File(lfn = "/singleLfn/%s/%s" % (name, n),
size = 1024, events = 10)
if type(site) == list:
for singleSite in site:
testFile.setLocation(singleSite)
else:
testFile.setLocation(site)
testFile.create()
fileset.addFile(testFile)
fileset.commit()
index = 0
for f in fileset.files:
index += 1
testJob = Job(name = '%s-%i' % (name, index))
testJob.addFile(f)
testJob["location"] = f.getLocations()[0]
testJob['task'] = task.getPathName()
testJob['sandbox'] = task.data.input.sandbox
testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl')
testJob['mask']['FirstEvent'] = 101
testJob["siteBlacklist"] = bl
testJob["siteWhitelist"] = wl
testJob['priority'] = 101
testJob['multicoreEnabled'] = False
testJob['numberOfCores'] = 1
jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub), 'Job_%i' % (index))
os.makedirs(jobCache)
testJob.create(jobGroup)
testJob['cache_dir'] = jobCache
testJob.save()
jobGroup.add(testJob)
output = open(os.path.join(jobCache, 'job.pkl'), 'w')
pickle.dump(testJob, output)
output.close()
return testJob, testFile
def testParentageByJob(self):
"""
_testParentageByJob_
Tests the DAO that assigns parentage by Job
"""
testWorkflow = Workflow(spec = 'hello', owner = "mnorman",
name = "wf001", task="basicWorkload/Production")
testWorkflow.create()
testFileset = Fileset(name = "TestFileset")
testFileset.create()
testSubscription = Subscription(fileset = testFileset, workflow = testWorkflow, type = "Processing", split_algo = "FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFileParentA = File(lfn = "/this/is/a/parent/lfnA", size = 1024,
events = 20, checksums = {'cksum': 1})
testFileParentA.addRun(Run( 1, *[45]))
testFileParentB = File(lfn = "/this/is/a/parent/lfnB", size = 1024,
events = 20, checksums = {'cksum': 1})
testFileParentB.addRun(Run( 1, *[45]))
testFileParentA.create()
testFileParentB.create()
testFileA = File(lfn = "/this/is/a/lfn", size = 1024, events = 10,
checksums = {'cksum':1})
testFileA.addRun(Run( 1, *[45]))
testFileA.create()
testJobA = Job()
testJobA.create(group = testJobGroup)
testJobA.addFile(testFileParentA)
testJobA.addFile(testFileParentB)
testJobA.associateFiles()
parentAction = self.daofactory(classname = "Files.SetParentageByJob")
parentAction.execute(binds = {'jobid': testJobA.exists(), 'child': testFileA['lfn']})
testFileB = File(id = testFileA["id"])
testFileB.loadData(parentage = 1)
goldenFiles = [testFileParentA, testFileParentB]
for parentFile in testFileB["parents"]:
self.assertEqual(parentFile in goldenFiles, True,
"ERROR: Unknown parent file")
goldenFiles.remove(parentFile)
self.assertEqual(len(goldenFiles), 0,
"ERROR: Some parents are missing")
def makeNJobs(self, name, task, nJobs, jobGroup, fileset, sub, site):
"""
_makeNJobs_
Make and return a WMBS Job and File
This handles all those damn add-ons
"""
# Set the CacheDir
cacheDir = os.path.join(self.testDir, 'CacheDir')
for n in range(nJobs):
# First make a file
#site = self.sites[0]
testFile = File(lfn = "/singleLfn/%s/%s" % (name, n),
size = 1024, events = 10)
fileset.addFile(testFile)
fileset.commit()
location = None
if isinstance(site, list):
if len(site) > 0:
location = site[0]
else:
location = site
index = 0
for f in fileset.files:
index += 1
testJob = Job(name = '%s-%i' % (name, index))
testJob.addFile(f)
testJob["location"] = location
testJob["possiblePSN"] = set(site) if isinstance(site, list) else set([site])
testJob['task'] = task.getPathName()
testJob['sandbox'] = task.data.input.sandbox
testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl')
testJob['mask']['FirstEvent'] = 101
testJob['priority'] = 101
testJob['numberOfCores'] = 1
jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub), 'Job_%i' % (index))
os.makedirs(jobCache)
testJob.create(jobGroup)
testJob['cache_dir'] = jobCache
testJob.save()
jobGroup.add(testJob)
output = open(os.path.join(jobCache, 'job.pkl'), 'w')
pickle.dump(testJob, output)
output.close()
return testJob, testFile
def createTestJobs(self, nJobs, cacheDir):
"""
_createTestJobs_
Create several jobs
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
# Create a file
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
baseName = makeUUID()
# Now create a job
for i in range(nJobs):
testJob = Job(name='%s-%i' % (baseName, i))
testJob.addFile(testFileA)
testJob['location'] = 'malpaquet'
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob.create(testJobGroup)
testJob.save()
testJobGroup.add(testJob)
testJobGroup.commit()
# Set test job caches
for job in testJobGroup.jobs:
job.setCache(cacheDir)
return testJobGroup
def createTestJobs(self, nJobs, cacheDir):
"""
_createTestJobs_
Create several jobs
"""
testWorkflow = Workflow(spec = "spec.xml", owner = "Simon",
name = "wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name = "TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow,
type = "Processing",
split_algo = "FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
# Create a file
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
baseName = makeUUID()
# Now create a job
for i in range(nJobs):
testJob = Job(name = '%s-%i' % (baseName, i))
testJob.addFile(testFileA)
testJob['location'] = 'malpaquet'
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob.create(testJobGroup)
testJob.save()
testJobGroup.add(testJob)
testJobGroup.commit()
# Set test job caches
for job in testJobGroup.jobs:
job.setCache(cacheDir)
return testJobGroup
def testTwoJobGroups(self):
"""
Test two job groups with a shared fileset. (Minimal part of testGetLocations which was failing)
"""
testWorkflow1 = Workflow(spec="spec.xml", owner="Simon", name="wf001", task="Test1")
testWorkflow1.create()
testWMBSFileset1 = WMBSFileset(name="TestFileset1")
testWMBSFileset1.create()
testSubscription1 = Subscription(fileset=testWMBSFileset1, workflow=testWorkflow1)
testSubscription1.create()
testJobGroup1 = JobGroup(subscription=testSubscription1)
testJobGroup1.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.create()
testJobA = Job(name="TestJobA")
testJobA.addFile(testFileA)
testJobGroup1.add(testJobA)
testJobGroup1.commit()
testWorkflow2 = Workflow(spec="spec.xml", owner="Simon", name="wf002", task="Test2")
testWorkflow2.create()
testWMBSFileset2 = WMBSFileset(name="TestFileset1")
testWMBSFileset2.create()
testSubscription2 = Subscription(fileset=testWMBSFileset2, workflow=testWorkflow2)
testSubscription2.create()
testJobGroup2 = JobGroup(subscription=testSubscription2)
testJobGroup2.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.create()
testJobA1 = Job(name="TestJobA1")
testJobA1.addFile(testFileC)
testJobGroup2.add(testJobA1)
testJobGroup2.commit()
def createTestJobGroup(self):
"""
Creates a group of several jobs
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, self.nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
testJobGroup.commit()
return testJobGroup
def createTestJobGroup(self, commitFlag=True):
"""
_createTestJobGroup_
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = WMBSFileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation("goodse.cern.ch")
testFileA.setLocation("malpaquet")
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation("goodse.cern.ch")
testFileB.setLocation("malpaquet")
testFileA.create()
testFileB.create()
testJobA = Job(name="TestJobA")
testJobA.addFile(testFileA)
testJobB = Job(name="TestJobB")
testJobB.addFile(testFileB)
testJobGroup.add(testJobA)
testJobGroup.add(testJobB)
if commitFlag:
testJobGroup.commit()
return testJobGroup
def createTestJobGroup(self, commitFlag=True):
"""
_createTestJobGroup_
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = WMBSFileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation("T2_CH_CERN")
testFileA.setLocation("malpaquet")
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation("T2_CH_CERN")
testFileB.setLocation("malpaquet")
testFileA.create()
testFileB.create()
testJobA = Job(name="TestJobA")
testJobA.addFile(testFileA)
testJobB = Job(name="TestJobB")
testJobB.addFile(testFileB)
testJobGroup.add(testJobA)
testJobGroup.add(testJobB)
if commitFlag:
testJobGroup.commit()
return testJobGroup
def createTestJobGroup(self):
"""
Creates a group of several jobs
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for _ in range(0, self.nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
testJobGroup.commit()
return testJobGroup
def createTestJobGroup(self, nJobs, subType="Processing", retryOnce=False):
"""
_createTestJobGroup_
Creates a group of several jobs
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name=makeUUID(),
task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow,
type=subType)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
testFileB.create()
for _ in range(0, nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['cache_dir'] = os.path.join(self.testDir, testJob['name'])
os.mkdir(testJob['cache_dir'])
testJobGroup.add(testJob)
testJobGroup.commit()
if retryOnce:
self.increaseRetry.execute(testJobGroup.jobs)
return testJobGroup
def createTestJobGroup(self, nJobs, subType = "Processing", retryOnce = False):
"""
_createTestJobGroup_
Creates a group of several jobs
"""
testWorkflow = Workflow(spec = "spec.xml", owner = "Simon",
name = makeUUID(), task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name = "TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow,
type = subType)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
testFileB.create()
for i in range(0, nJobs):
testJob = Job(name = makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['cache_dir'] = os.path.join(self.testDir, testJob['name'])
os.mkdir(testJob['cache_dir'])
testJobGroup.add(testJob)
testJobGroup.commit()
if retryOnce:
self.increaseRetry.execute(testJobGroup.jobs)
return testJobGroup
def createLargerTestJobGroup(self, commitFlag=True):
"""
_createTestJobGroup_
"""
testWorkflow = Workflow(spec="spec.xml",
owner="Simon",
name="wf001",
task="Test")
testWorkflow.create()
testWMBSFileset = WMBSFileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation("goodse.cern.ch")
testFileC.setLocation("malpaquet")
testFileD = File(lfn="/this/is/a/lfnD", size=1024, events=10)
testFileD.addRun(Run(10, *[12312]))
testFileD.setLocation("goodse.cern.ch")
testFileD.setLocation("malpaquet")
testFileC.create()
testFileD.create()
testJobA = Job(name="TestJobA1")
testJobA.addFile(testFileC)
testJobB = Job(name="TestJobB1")
testJobB.addFile(testFileD)
testJobGroup.add(testJobA)
testJobGroup.add(testJobB)
for i in range(0, 100):
testJob = Job(name="TestJob%i" % (i))
testJob.addFile(testFileC)
testJobGroup.add(testJob)
if commitFlag:
testJobGroup.commit()
return testJobGroup
def createLargerTestJobGroup(self, commitFlag=True):
"""
_createTestJobGroup_
"""
testWorkflow = Workflow(spec="spec.xml", owner="Simon",
name="wf001", task="Test")
testWorkflow.create()
testWMBSFileset = WMBSFileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation("T2_CH_CERN")
testFileC.setLocation("malpaquet")
testFileD = File(lfn="/this/is/a/lfnD", size=1024, events=10)
testFileD.addRun(Run(10, *[12312]))
testFileD.setLocation("T2_CH_CERN")
testFileD.setLocation("malpaquet")
testFileC.create()
testFileD.create()
testJobA = Job(name="TestJobA1")
testJobA.addFile(testFileC)
testJobB = Job(name="TestJobB1")
testJobB.addFile(testFileD)
testJobGroup.add(testJobA)
testJobGroup.add(testJobB)
for i in range(0, 100):
testJob = Job(name="TestJob%i" % (i))
testJob.addFile(testFileC)
testJobGroup.add(testJob)
if commitFlag:
testJobGroup.commit()
return testJobGroup
def createGiantJobSet(self,
name,
config,
nSubs=10,
nJobs=10,
nFiles=1,
spec="spec.xml"):
"""
Creates a massive set of jobs
"""
jobList = []
for i in range(0, nSubs):
# Make a bunch of subscriptions
localName = '%s-%i' % (name, i)
testWorkflow = Workflow(spec=spec,
owner=self.OWNERDN,
name=localName,
task="Test",
owner_vogroup="",
owner_vorole="")
testWorkflow.create()
testWMBSFileset = Fileset(name=localName)
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
filesToComplete = []
for j in range(0, nJobs):
# Create jobs for each subscription
testFileA = File(lfn="%s-%i-lfnA" % (localName, j),
size=1024,
events=10)
testFileA.addRun(
Run(
10, *[
11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23,
24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36,
37, 38, 39, 40
]))
testFileA.setLocation('malpaquet')
testFileA.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.commit()
filesToComplete.append(testFileA)
testJob = Job(name='%s-%i' % (localName, j))
testJob.addFile(testFileA)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
jobList.append(testJob)
for k in range(0, nFiles):
# Create output files
testFile = File(lfn="%s-%i-output" % (localName, k),
size=1024,
events=10)
testFile.addRun(Run(10, *[12312]))
testFile.setLocation('malpaquet')
testFile.create()
testJobGroup.output.addFile(testFile)
testJobGroup.output.commit()
testJobGroup.commit()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'success', 'complete')
changer.propagate(testJobGroup.jobs, 'cleanout', 'success')
testWMBSFileset.markOpen(0)
testSubscription.completeFiles(filesToComplete)
return jobList
def testParentageByJob(self):
"""
_testParentageByJob_
Tests the DAO that assigns parentage by Job
"""
testWorkflow = Workflow(spec='hello',
owner="mnorman",
name="wf001",
task="basicWorkload/Production")
testWorkflow.create()
testFileset = Fileset(name="TestFileset")
testFileset.create()
testSubscription = Subscription(fileset=testFileset,
workflow=testWorkflow,
type="Processing",
split_algo="FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFileParentA = File(lfn="/this/is/a/parent/lfnA",
size=1024,
events=20,
checksums={'cksum': 1})
testFileParentA.addRun(Run(1, *[45]))
testFileParentB = File(lfn="/this/is/a/parent/lfnB",
size=1024,
events=20,
checksums={'cksum': 1})
testFileParentB.addRun(Run(1, *[45]))
testFileParentA.create()
testFileParentB.create()
testFileA = File(lfn="/this/is/a/lfn",
size=1024,
events=10,
checksums={'cksum': 1})
testFileA.addRun(Run(1, *[45]))
testFileA.create()
testJobA = Job()
testJobA.create(group=testJobGroup)
testJobA.addFile(testFileParentA)
testJobA.addFile(testFileParentB)
testJobA.associateFiles()
parentAction = self.daofactory(classname="Files.SetParentageByJob")
parentAction.execute(binds={
'jobid': testJobA.exists(),
'child': testFileA['lfn']
})
testFileB = File(id=testFileA["id"])
testFileB.loadData(parentage=1)
goldenFiles = [testFileParentA, testFileParentB]
for parentFile in testFileB["parents"]:
self.assertEqual(parentFile in goldenFiles, True,
"ERROR: Unknown parent file")
goldenFiles.remove(parentFile)
self.assertEqual(len(goldenFiles), 0,
"ERROR: Some parents are missing")
def createGiantJobSet(self, name, config, nSubs = 10, nJobs = 10,
nFiles = 1, spec = "spec.xml"):
"""
Creates a massive set of jobs
"""
jobList = []
for i in range(0, nSubs):
# Make a bunch of subscriptions
localName = '%s-%i' % (name, i)
testWorkflow = Workflow(spec = spec, owner = self.OWNERDN,
name = localName, task="Test", owner_vogroup="", owner_vorole="")
testWorkflow.create()
testWMBSFileset = Fileset(name = localName)
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
filesToComplete = []
for j in range(0, nJobs):
# Create jobs for each subscription
testFileA = File(lfn = "%s-%i-lfnA" % (localName, j) , size = 1024, events = 10)
testFileA.addRun(Run(10, *[11,12,13,14,15,16,17,18,19,20,
21,22,23,24,25,26,27,28,29,30,
31,32,33,34,35,36,37,38,39,40]))
testFileA.setLocation('malpaquet')
testFileA.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.commit()
filesToComplete.append(testFileA)
testJob = Job(name = '%s-%i' % (localName, j))
testJob.addFile(testFileA)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJobGroup.add(testJob)
jobList.append(testJob)
for k in range(0, nFiles):
# Create output files
testFile = File(lfn = "%s-%i-output" % (localName, k) , size = 1024, events = 10)
testFile.addRun(Run(10, *[12312]))
testFile.setLocation('malpaquet')
testFile.create()
testJobGroup.output.addFile(testFile)
testJobGroup.output.commit()
testJobGroup.commit()
changer = ChangeState(config)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'success', 'complete')
changer.propagate(testJobGroup.jobs, 'cleanout', 'success')
testWMBSFileset.markOpen(0)
testSubscription.completeFiles(filesToComplete)
return jobList
def createTestJobGroup(self, config, name = "TestWorkthrough",
filesetName = "TestFileset",
specLocation = "spec.xml", error = False,
task = "/TestWorkload/ReReco",
type = "Processing"):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec = specLocation, owner = self.OWNERDN,
name = name, task = task, owner_vogroup="", owner_vorole="")
testWorkflow.create()
self.inject.execute(names = [name], injected = True)
testWMBSFileset = Fileset(name = filesetName)
testWMBSFileset.create()
testFileA = File(lfn = "/this/is/a/lfnA" , size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
outputWMBSFileset = Fileset(name = '%sOutput' % filesetName)
outputWMBSFileset.create()
testFileC = File(lfn = "/this/is/a/lfnC" , size = 1024, events = 10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation('malpaquet')
testFileC.create()
outputWMBSFileset.addFile(testFileC)
outputWMBSFileset.commit()
outputWMBSFileset.markOpen(0)
testWorkflow.addOutput('output', outputWMBSFileset)
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow,
type = type)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
for i in range(0,self.nJobs):
testJob = Job(name = makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
changer = ChangeState(config)
report1 = Report()
report2 = Report()
if error:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl")
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
else:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'mergeReport1.pkl')
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
report1.load(filename = path1)
report2.load(filename = path2)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
for i in range(self.nJobs):
if i < self.nJobs/2:
testJobGroup.jobs[i]['fwjr'] = report1
else:
testJobGroup.jobs[i]['fwjr'] = report2
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'retrydone', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'exhausted', 'retrydone')
changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def createTestJob(self):
"""
_createTestJobs_
Create several jobs
"""
#Create sites in resourceControl
resourceControl = ResourceControl()
resourceControl.insertSite(siteName = 'malpaquet', seName = 'se.malpaquet',
ceName = 'malpaquet', plugin = "CondorPlugin")
resourceControl.insertThreshold(siteName = 'malpaquet', taskType = 'Processing', \
maxSlots = 10000)
locationAction = self.daoFactory(classname = "Locations.New")
locationAction.execute(siteName = "malpaquet", seName = "malpaquet",
ceName = "malpaquet", plugin = "CondorPlugin")
jobAction = self.daoFactory(classname = "Jobs.New")
jobAction.execute()
# Create user
newuser = self.daoFactory(classname = "Users.New")
newuser.execute(dn = "jchurchill")
testWorkflow = Workflow(spec = "spec.xml", owner = "Simon",
name = "meloam_ASYNCTEST1_120810_170823_8981", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name = "TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow,
type = "Processing",
split_algo = "FileBased")
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
# Create a file
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileA.create()
baseName = makeUUID()
# make a copy of the FWJR since it will be modified
shutil.copyfile(os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t',
'AsyncStageoutTracker_t',
'oneaso.pkl'),
os.path.join(self.testDir, "oneaso.pkl") )
# Now create a job
testJob = Job(name = '%s-%i' % (baseName, 1))
testJob.addFile(testFileA)
testJob['location'] = 'malpaquet'
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['fwjr_path'] = os.path.join(self.testDir, "oneaso.pkl")
testJob.create(testJobGroup)
testJob.save()
testJobGroup.add(testJob)
testJobGroup.commit()
stateAction = self.daoFactory(classname = "Jobs.ChangeState")
stateAction.execute( [{'id' : testJob['id'],
'state' : 'asopending',
'couch_record' : testJob['couch_record']}])
return testJob
return testJobGroup
def createTestJobGroup(
self, config, name="TestWorkthrough", specLocation="spec.xml", error=False, task="/TestWorkload/ReReco"
):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=specLocation, owner="Simon", name=name, task=task)
testWorkflow.create()
testWMBSFileset = Fileset(name=name)
testWMBSFileset.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation("malpaquet")
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation("malpaquet")
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
testSubscription = Subscription(fileset=testWMBSFileset, workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, self.nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob["retry_count"] = 1
testJob["retry_max"] = 10
testJob["mask"].addRunAndLumis(run=10, lumis=[12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
changer = ChangeState(config)
report1 = Report()
report2 = Report()
if error:
path1 = os.path.join(
WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl"
)
path2 = path1
else:
path1 = os.path.join(
WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/TaskArchiver_t/fwjrs", "mergeReport1.pkl"
)
path2 = os.path.join(
WMCore.WMInit.getWMBASE(), "test/python/WMComponent_t/TaskArchiver_t/fwjrs", "mergeReport2.pkl"
)
report1.load(filename=path1)
report2.load(filename=path2)
changer.propagate(testJobGroup.jobs, "created", "new")
changer.propagate(testJobGroup.jobs, "executing", "created")
changer.propagate(testJobGroup.jobs, "complete", "executing")
for i in range(self.nJobs):
if i < self.nJobs / 2:
testJobGroup.jobs[i]["fwjr"] = report1
else:
testJobGroup.jobs[i]["fwjr"] = report2
changer.propagate(testJobGroup.jobs, "jobfailed", "complete")
changer.propagate(testJobGroup.jobs, "exhausted", "jobfailed")
changer.propagate(testJobGroup.jobs, "cleanout", "exhausted")
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def createTestJobGroup(self, nJobs = 10, retry_count = 1,
workloadPath = 'test', fwjrPath = None,
workloadName = makeUUID()):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testWorkflow = Workflow(spec = workloadPath, owner = "cmsdataops", group = "cmsdataops",
name = workloadName, task="/TestWorkload/ReReco")
testWorkflow.create()
testWMBSFileset = Fileset(name = "TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFile0 = File(lfn = "/this/is/a/parent", size = 1024, events = 10)
testFile0.addRun(Run(10, *[12312]))
testFile0.setLocation('malpaquet')
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10,
first_event = 88, last_event = 99)
testFileA.addRun(Run(10, *[12312, 12313]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10,
first_event = 88, last_event = 99)
testFileB.addRun(Run(10, *[12314, 12315, 12316]))
testFileB.setLocation('malpaquet')
testFile0.create()
testFileA.create()
testFileB.create()
testFileA.addParent(lfn = "/this/is/a/parent")
testFileB.addParent(lfn = "/this/is/a/parent")
for i in range(0, nJobs):
testJob = Job(name = makeUUID())
testJob['retry_count'] = retry_count
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run = 10, lumis = [12312])
testJob['mask'].addRunAndLumis(run = 10, lumis = [12314, 12316])
testJob['mask']['FirstEvent'] = 100
testJob['cache_dir'] = os.path.join(self.testDir, testJob['name'])
testJob['fwjr_path'] = fwjrPath
os.mkdir(testJob['cache_dir'])
testJobGroup.add(testJob)
testJob.create(group = testJobGroup)
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob.save()
testJobGroup.commit()
testSubscription.acquireFiles(files = [testFileA, testFileB])
testSubscription.save()
myThread.transaction.commit()
return testJobGroup
def createTestJobGroup(self,
nJobs=10,
retry_count=1,
workloadPath='test',
fwjrPath=None,
workloadName=makeUUID(),
fileModifier=''):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testWorkflow = Workflow(spec=workloadPath,
owner="cmsdataops",
group="cmsdataops",
name=workloadName,
task="/TestWorkload/ReReco")
testWorkflow.create()
testWMBSFileset = Fileset(name="TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
testFile0 = File(lfn="/this/is/a/parent%s" % fileModifier,
size=1024,
events=10)
testFile0.addRun(Run(10, *[12312]))
testFile0.setLocation('T2_CH_CERN')
testFileA = File(lfn="/this/is/a/lfnA%s" % fileModifier,
size=1024,
events=10,
first_event=88,
merged=False)
testFileA.addRun(Run(10, *[12312, 12313]))
testFileA.setLocation('T2_CH_CERN')
testFileB = File(lfn="/this/is/a/lfnB%s" % fileModifier,
size=1024,
events=10,
first_event=88,
merged=False)
testFileB.addRun(Run(10, *[12314, 12315, 12316]))
testFileB.setLocation('T2_CH_CERN')
testFile0.create()
testFileA.create()
testFileB.create()
testFileA.addParent(lfn="/this/is/a/parent%s" % fileModifier)
testFileB.addParent(lfn="/this/is/a/parent%s" % fileModifier)
for i in range(0, nJobs):
testJob = Job(name=makeUUID())
testJob['retry_count'] = retry_count
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run=10, lumis=[12312])
testJob['mask'].addRunAndLumis(run=10, lumis=[12314, 12316])
testJob['cache_dir'] = os.path.join(self.testDir, testJob['name'])
testJob['fwjr_path'] = fwjrPath
os.mkdir(testJob['cache_dir'])
testJobGroup.add(testJob)
testJob.create(group=testJobGroup)
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob.save()
testJobGroup.commit()
testSubscription.acquireFiles(files=[testFileA, testFileB])
testSubscription.save()
myThread.transaction.commit()
return testJobGroup
def createTestJobGroup(self,
config,
name="TestWorkthrough",
filesetName="TestFileset",
specLocation="spec.xml",
error=False,
task="/TestWorkload/ReReco",
multicore=False):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=specLocation,
owner=self.OWNERDN,
name=name,
task=task,
owner_vogroup="",
owner_vorole="")
testWorkflow.create()
self.inject.execute(names=[name], injected=True)
testWMBSFileset = Fileset(name=filesetName)
testWMBSFileset.create()
testFileA = File(lfn="/this/is/a/lfnA", size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn="/this/is/a/lfnB", size=1024, events=10)
testFileB.addRun(Run(10, *[12314]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
outputWMBSFileset = Fileset(name='%sOutput' % filesetName)
outputWMBSFileset.create()
testFileC = File(lfn="/this/is/a/lfnC", size=1024, events=10)
testFileC.addRun(Run(10, *[12312]))
testFileC.setLocation('malpaquet')
testFileC.create()
outputWMBSFileset.addFile(testFileC)
outputWMBSFileset.commit()
outputWMBSFileset.markOpen(0)
testWorkflow.addOutput('output', outputWMBSFileset)
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, self.nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
changer = ChangeState(config)
report1 = Report()
report2 = Report()
if error:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"badBackfillJobReport.pkl")
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
elif multicore:
path1 = os.path.join(
WMCore.WMBase.getTestBase(),
"WMCore_t/FwkJobReport_t/MulticoreReport.pkl")
path2 = path1
else:
path1 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'mergeReport1.pkl')
path2 = os.path.join(WMCore.WMBase.getTestBase(),
'WMComponent_t/TaskArchiver_t/fwjrs',
'logCollectReport2.pkl')
report1.load(filename=path1)
report2.load(filename=path2)
changer.propagate(testJobGroup.jobs, 'created', 'new')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
for i in range(self.nJobs):
if i < self.nJobs / 2:
testJobGroup.jobs[i]['fwjr'] = report1
else:
testJobGroup.jobs[i]['fwjr'] = report2
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'jobcooloff', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'created', 'jobcooloff')
changer.propagate(testJobGroup.jobs, 'executing', 'created')
changer.propagate(testJobGroup.jobs, 'complete', 'executing')
changer.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
changer.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed')
changer.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def stuffWMBS(self, injected=True):
"""
_stuffWMBS_
Insert some dummy jobs, jobgroups, filesets, files and subscriptions
into WMBS to test job creation. Three completed job groups each
containing several files are injected. Another incomplete job group is
also injected. Also files are added to the "Mergeable" subscription as
well as to the output fileset for their jobgroups.
"""
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="T2_CH_CERN", pnn="T2_CH_CERN")
locationAction.execute(siteName="T1_US_FNAL", pnn="T2_CH_CERN")
changeStateDAO = self.daoFactory(classname="Jobs.ChangeState")
self.mergeFileset = Fileset(name="mergeFileset")
self.mergeFileset.create()
self.bogusFileset = Fileset(name="bogusFileset")
self.bogusFileset.create()
self.mergeMergedFileset = Fileset(name="mergeMergedFileset")
self.mergeMergedFileset.create()
self.bogusMergedFileset = Fileset(name="bogusMergedFileset")
self.bogusMergedFileset.create()
mergeWorkflow = Workflow(name="mergeWorkflow", spec="bunk2",
owner="Steve", task="Test")
mergeWorkflow.create()
markWorkflow = self.daoFactory(classname="Workflow.MarkInjectedWorkflows")
markWorkflow.execute(names=[mergeWorkflow.name], injected=injected)
self.mergeSubscription = Subscription(fileset=self.mergeFileset,
workflow=mergeWorkflow,
split_algo="WMBSMergeBySize")
self.mergeSubscription.create()
self.bogusSubscription = Subscription(fileset=self.bogusFileset,
workflow=mergeWorkflow,
split_algo="WMBSMergeBySize")
inputFileset = Fileset(name="inputFileset")
inputFileset.create()
inputWorkflow = Workflow(name="inputWorkflow", spec="input",
owner="Steve", task="Test")
inputWorkflow.create()
inputWorkflow.addOutput("output", self.mergeFileset,
self.mergeMergedFileset)
inputWorkflow.addOutput("output2", self.bogusFileset,
self.bogusMergedFileset)
bogusInputWorkflow = Workflow(name="bogusInputWorkflow", spec="input",
owner="Steve", task="Test")
bogusInputWorkflow.create()
inputSubscription = Subscription(fileset=inputFileset,
workflow=inputWorkflow)
inputSubscription.create()
bogusInputSubscription = Subscription(fileset=inputFileset,
workflow=bogusInputWorkflow)
bogusInputSubscription.create()
parentFile1 = File(lfn="parentFile1")
parentFile1.create()
parentFile2 = File(lfn="parentFile2")
parentFile2.create()
parentFile3 = File(lfn="parentFile3")
parentFile3.create()
parentFile4 = File(lfn="parentFile4")
parentFile4.create()
self.parentFileSite2 = File(lfn="parentFileSite2")
self.parentFileSite2.create()
jobGroup1 = JobGroup(subscription=inputSubscription)
jobGroup1.create()
jobGroup2 = JobGroup(subscription=inputSubscription)
jobGroup2.create()
jobGroup3 = JobGroup(subscription=bogusInputSubscription)
jobGroup3.create()
testJob1 = Job()
testJob1.addFile(parentFile1)
testJob1.create(jobGroup1)
testJob1["state"] = "cleanout"
testJob1["oldstate"] = "new"
testJob1["couch_record"] = "somejive"
testJob1["retry_count"] = 0
testJob1["outcome"] = "success"
testJob1.save()
changeStateDAO.execute([testJob1])
testJob1A = Job()
testJob1A.addFile(parentFile1)
testJob1A.create(jobGroup3)
testJob1A["state"] = "cleanout"
testJob1A["oldstate"] = "new"
testJob1A["couch_record"] = "somejive"
testJob1A["retry_count"] = 0
testJob1A["outcome"] = "failure"
testJob1A.save()
changeStateDAO.execute([testJob1A])
testJob2 = Job()
testJob2.addFile(parentFile2)
testJob2.create(jobGroup1)
testJob2["state"] = "cleanout"
testJob2["oldstate"] = "new"
testJob2["couch_record"] = "somejive"
testJob2["retry_count"] = 0
testJob2["outcome"] = "success"
testJob2.save()
changeStateDAO.execute([testJob2])
testJob3 = Job()
testJob3.addFile(parentFile3)
testJob3.create(jobGroup2)
testJob3["state"] = "cleanout"
testJob3["oldstate"] = "new"
testJob3["couch_record"] = "somejive"
testJob3["retry_count"] = 0
testJob3["outcome"] = "success"
testJob3.save()
changeStateDAO.execute([testJob3])
testJob4 = Job()
testJob4.addFile(parentFile4)
testJob4.create(jobGroup2)
testJob4["state"] = "cleanout"
testJob4["oldstate"] = "new"
testJob4["couch_record"] = "somejive"
testJob4["retry_count"] = 0
testJob4["outcome"] = "failure"
testJob4.save()
changeStateDAO.execute([testJob4])
# We'll simulate a failed split by event job that the merger should
# ignore.
parentFile5 = File(lfn="parentFile5")
parentFile5.create()
testJob5 = Job()
testJob5.addFile(parentFile5)
testJob5.create(jobGroup2)
testJob5["state"] = "cleanout"
testJob5["oldstate"] = "new"
testJob5["couch_record"] = "somejive"
testJob5["retry_count"] = 0
testJob5["outcome"] = "success"
testJob5.save()
changeStateDAO.execute([testJob5])
testJob6 = Job()
testJob6.addFile(parentFile5)
testJob6.create(jobGroup2)
testJob6["state"] = "cleanout"
testJob6["oldstate"] = "new"
testJob6["couch_record"] = "somejive"
testJob6["retry_count"] = 0
testJob6["outcome"] = "failure"
testJob6.save()
changeStateDAO.execute([testJob6])
testJob7 = Job()
testJob7.addFile(self.parentFileSite2)
testJob7.create(jobGroup2)
testJob7["state"] = "cleanout"
testJob7["oldstate"] = "new"
testJob7["couch_record"] = "somejive"
testJob7["retry_count"] = 0
testJob7["outcome"] = "success"
testJob7.save()
changeStateDAO.execute([testJob7])
badFile1 = File(lfn="badFile1", size=10241024, events=10241024,
first_event=0, locations={"T2_CH_CERN"})
badFile1.addRun(Run(1, *[45]))
badFile1.create()
badFile1.addParent(parentFile5["lfn"])
file1 = File(lfn="file1", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
file1.addRun(Run(1, *[45]))
file1.create()
file1.addParent(parentFile1["lfn"])
file2 = File(lfn="file2", size=1024, events=1024,
first_event=1024, locations={"T2_CH_CERN"})
file2.addRun(Run(1, *[45]))
file2.create()
file2.addParent(parentFile1["lfn"])
file3 = File(lfn="file3", size=1024, events=1024,
first_event=2048, locations={"T2_CH_CERN"})
file3.addRun(Run(1, *[45]))
file3.create()
file3.addParent(parentFile1["lfn"])
file4 = File(lfn="file4", size=1024, events=1024,
first_event=3072, locations={"T2_CH_CERN"})
file4.addRun(Run(1, *[45]))
file4.create()
file4.addParent(parentFile1["lfn"])
fileA = File(lfn="fileA", size=1024, events=1024,
first_event=0, locations={"T2_CH_CERN"})
fileA.addRun(Run(1, *[46]))
fileA.create()
fileA.addParent(parentFile2["lfn"])
fileB = File(lfn="fileB", size=1024, events=1024,
first_event=1024, locations={"T2_CH_CERN"})
fileB.addRun(Run(1, *[46]))
fileB.create()
fileB.addParent(parentFile2["lfn"])
fileC = File(lfn="fileC", size=1024, events=1024,
first_event=2048, locations={"T2_CH_CERN"})
fileC.addRun(Run(1, *[46]))
fileC.create()
fileC.addParent(parentFile2["lfn"])
fileI = File(lfn="fileI", size=1024, events=1024,
first_event=0, locations={"T2_CH_CERN"})
fileI.addRun(Run(2, *[46]))
fileI.create()
fileI.addParent(parentFile3["lfn"])
fileII = File(lfn="fileII", size=1024, events=1024,
first_event=1024, locations={"T2_CH_CERN"})
fileII.addRun(Run(2, *[46]))
fileII.create()
fileII.addParent(parentFile3["lfn"])
fileIII = File(lfn="fileIII", size=1024, events=1024,
first_event=2048, locations={"T2_CH_CERN"})
fileIII.addRun(Run(2, *[46]))
fileIII.create()
fileIII.addParent(parentFile3["lfn"])
fileIV = File(lfn="fileIV", size=1024, events=1024,
first_event=3072, locations={"T2_CH_CERN"})
fileIV.addRun(Run(2, *[46]))
fileIV.create()
fileIV.addParent(parentFile3["lfn"])
fileX = File(lfn="badFileA", size=1024, events=1024,
first_event=0, locations={"T2_CH_CERN"})
fileX.addRun(Run(1, *[47]))
fileX.create()
fileX.addParent(parentFile4["lfn"])
fileY = File(lfn="badFileB", size=1024, events=1024,
first_event=1024, locations={"T2_CH_CERN"})
fileY.addRun(Run(1, *[47]))
fileY.create()
fileY.addParent(parentFile4["lfn"])
fileZ = File(lfn="badFileC", size=1024, events=1024,
first_event=2048, locations={"T2_CH_CERN"})
fileZ.addRun(Run(1, *[47]))
fileZ.create()
fileZ.addParent(parentFile4["lfn"])
jobGroup1.output.addFile(file1)
jobGroup1.output.addFile(file2)
jobGroup1.output.addFile(file3)
jobGroup1.output.addFile(file4)
jobGroup1.output.addFile(fileA)
jobGroup1.output.addFile(fileB)
jobGroup1.output.addFile(fileC)
jobGroup1.output.commit()
jobGroup2.output.addFile(fileI)
jobGroup2.output.addFile(fileII)
jobGroup2.output.addFile(fileIII)
jobGroup2.output.addFile(fileIV)
jobGroup2.output.addFile(fileX)
jobGroup2.output.addFile(fileY)
jobGroup2.output.addFile(fileZ)
jobGroup2.output.addFile(badFile1)
jobGroup2.output.commit()
for fileObj in [file1, file2, file3, file4, fileA, fileB, fileC, fileI,
fileII, fileIII, fileIV, fileX, fileY, fileZ, badFile1]:
self.mergeFileset.addFile(fileObj)
self.bogusFileset.addFile(fileObj)
self.mergeFileset.commit()
self.bogusFileset.commit()
return
def createTestJobGroup(self, nJobs = 10, retry_count = 0, workloadPath = 'test'):
"""
Creates a group of several jobs
"""
myThread = threading.currentThread()
myThread.transaction.begin()
testWorkflow = Workflow(spec = workloadPath, owner = "Simon",
name = "wf001", task="Test")
testWorkflow.create()
testWMBSFileset = Fileset(name = "TestFileset")
testWMBSFileset.create()
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
testFile0 = File(lfn = "/this/is/a/parent", size = 1024, events = 10)
testFile0.addRun(Run(10, *[12312]))
testFile0.setLocation('malpaquet')
testFileA = File(lfn = "/this/is/a/lfnA", size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = "/this/is/a/lfnB", size = 1024, events = 10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation('malpaquet')
testFile0.create()
testFileA.create()
testFileB.create()
testFileA.addParent(lfn = "/this/is/a/parent")
testFileB.addParent(lfn = "/this/is/a/parent")
for i in range(0, nJobs):
testJob = Job(name = makeUUID())
testJob['retry_count'] = retry_count
testJob['retry_max'] = 10
testJob['group'] = 'BadGuys'
testJob['user'] = '******'
testJob['taskType'] = 'Merge'
#testJob['fwjr'] = myReport
testJobGroup.add(testJob)
testJob.create(group = testJobGroup)
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob.save()
testJobGroup.commit()
testSubscription.acquireFiles(files = [testFileA, testFileB])
testSubscription.save()
myThread.transaction.commit()
return testJobGroup
def testParallelProcessing(self):
"""
_testParallelProcessing_
Verify that merging works correctly when multiple processing
subscriptions are run over the same input files. The merging algorithm
should ignore processing jobs that feed into different merge
subscriptions.
"""
locationAction = self.daoFactory(classname = "Locations.New")
locationAction.execute(siteName = "s1", seName = "somese.cern.ch")
mergeFilesetA = Fileset(name = "mergeFilesetA")
mergeFilesetB = Fileset(name = "mergeFilesetB")
mergeFilesetA.create()
mergeFilesetB.create()
mergeMergedFilesetA = Fileset(name = "mergeMergedFilesetA")
mergeMergedFilesetB = Fileset(name = "mergeMergedFilesetB")
mergeMergedFilesetA.create()
mergeMergedFilesetB.create()
mergeWorkflow = Workflow(name = "mergeWorkflow", spec = "bogus",
owner = "Steve", task = "Test")
mergeWorkflow.create()
mergeSubscriptionA = Subscription(fileset = mergeFilesetA,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize")
mergeSubscriptionB = Subscription(fileset = mergeFilesetB,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize")
mergeSubscriptionA.create()
mergeSubscriptionB.create()
inputFileset = Fileset(name = "inputFileset")
inputFileset.create()
inputFileA = File(lfn = "inputLFNA")
inputFileB = File(lfn = "inputLFNB")
inputFileA.create()
inputFileB.create()
procWorkflowA = Workflow(name = "procWorkflowA", spec = "bunk2",
owner = "Steve", task = "Test")
procWorkflowA.create()
procWorkflowA.addOutput("output", mergeFilesetA, mergeMergedFilesetA)
procWorkflowB = Workflow(name = "procWorkflowB", spec = "bunk3",
owner = "Steve", task = "Test2")
procWorkflowB.create()
procWorkflowB.addOutput("output", mergeFilesetB, mergeMergedFilesetB)
procSubscriptionA = Subscription(fileset = inputFileset,
workflow = procWorkflowA,
split_algo = "EventBased")
procSubscriptionA.create()
procSubscriptionB = Subscription(fileset = inputFileset,
workflow = procWorkflowB,
split_algo = "EventBased")
procSubscriptionB.create()
jobGroupA = JobGroup(subscription = procSubscriptionA)
jobGroupA.create()
jobGroupB = JobGroup(subscription = procSubscriptionB)
jobGroupB.create()
changeStateDAO = self.daoFactory(classname = "Jobs.ChangeState")
testJobA = Job()
testJobA.addFile(inputFileA)
testJobA.create(jobGroupA)
testJobA["state"] = "cleanout"
testJobA["oldstate"] = "new"
testJobA["couch_record"] = "somejive"
testJobA["retry_count"] = 0
testJobA["outcome"] = "success"
testJobA.save()
testJobB = Job()
testJobB.addFile(inputFileB)
testJobB.create(jobGroupA)
testJobB["state"] = "cleanout"
testJobB["oldstate"] = "new"
testJobB["couch_record"] = "somejive"
testJobB["retry_count"] = 0
testJobB["outcome"] = "success"
testJobB.save()
testJobC = Job()
testJobC.addFile(inputFileA)
testJobC.create(jobGroupB)
testJobC["state"] = "cleanout"
testJobC["oldstate"] = "new"
testJobC["couch_record"] = "somejive"
testJobC["retry_count"] = 0
testJobC["outcome"] = "success"
testJobC.save()
testJobD = Job()
testJobD.addFile(inputFileA)
testJobD.create(jobGroupB)
testJobD["state"] = "cleanout"
testJobD["oldstate"] = "new"
testJobD["couch_record"] = "somejive"
testJobD["retry_count"] = 0
testJobD["outcome"] = "failure"
testJobD.save()
testJobE = Job()
testJobE.addFile(inputFileB)
testJobE.create(jobGroupB)
testJobE["state"] = "cleanout"
testJobE["oldstate"] = "new"
testJobE["couch_record"] = "somejive"
testJobE["retry_count"] = 0
testJobE["outcome"] = "success"
testJobE.save()
testJobF = Job()
testJobF.addFile(inputFileB)
testJobF.create(jobGroupB)
testJobF["state"] = "cleanout"
testJobF["oldstate"] = "new"
testJobF["couch_record"] = "somejive"
testJobF["retry_count"] = 0
testJobF["outcome"] = "failure"
testJobF.save()
changeStateDAO.execute([testJobA, testJobB, testJobC, testJobD,
testJobE, testJobF])
fileA = File(lfn = "fileA", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
fileA.addRun(Run(1, *[45]))
fileA.create()
fileA.addParent(inputFileA["lfn"])
fileB = File(lfn = "fileB", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
fileB.addRun(Run(1, *[45]))
fileB.create()
fileB.addParent(inputFileB["lfn"])
jobGroupA.output.addFile(fileA)
jobGroupA.output.addFile(fileB)
jobGroupA.output.commit()
mergeFilesetA.addFile(fileA)
mergeFilesetA.addFile(fileB)
mergeFilesetA.commit()
fileC = File(lfn = "fileC", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
fileC.addRun(Run(1, *[45]))
fileC.create()
fileC.addParent(inputFileA["lfn"])
fileD = File(lfn = "fileD", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
fileD.addRun(Run(1, *[45]))
fileD.create()
fileD.addParent(inputFileB["lfn"])
jobGroupB.output.addFile(fileC)
jobGroupB.output.addFile(fileD)
mergeFilesetB.addFile(fileC)
mergeFilesetB.addFile(fileD)
mergeFilesetB.commit()
splitter = SplitterFactory()
jobFactory = splitter(package = "WMCore.WMBS",
subscription = mergeSubscriptionB)
result = jobFactory(min_merge_size = 1, max_merge_size = 20000,
max_merge_events = 7169)
assert len(result) == 0, \
"Error: No merge jobs should have been created."
fileE = File(lfn = "fileE", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
fileE.addRun(Run(1, *[45]))
fileE.create()
fileE.addParent(inputFileA["lfn"])
fileF = File(lfn = "fileF", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
fileF.addRun(Run(1, *[45]))
fileF.create()
fileF.addParent(inputFileB["lfn"])
jobGroupB.output.addFile(fileE)
jobGroupB.output.addFile(fileF)
mergeFilesetB.addFile(fileE)
mergeFilesetB.addFile(fileF)
mergeFilesetB.commit()
testJobD["outcome"] = "success"
testJobD.save()
testJobF["outcome"] = "success"
testJobF.save()
changeStateDAO.execute([testJobD, testJobF])
result = jobFactory(min_merge_size = 1, max_merge_size = 20000,
max_merge_events = 7169)
assert len(result) == 1, \
"Error: One merge job should have been created: %s" % len(result)
return
def testParallelProcessing(self):
"""
_testParallelProcessing_
Verify that merging works correctly when multiple processing
subscriptions are run over the same input files. The merging algorithm
should ignore processing jobs that feed into different merge
subscriptions.
"""
locationAction = self.daoFactory(classname="Locations.New")
locationAction.execute(siteName="T2_CH_CERN", pnn="T2_CH_CERN")
locationAction.execute(siteName="T1_US_FNAL", pnn="T2_CH_CERN")
mergeFilesetA = Fileset(name="mergeFilesetA")
mergeFilesetB = Fileset(name="mergeFilesetB")
mergeFilesetA.create()
mergeFilesetB.create()
mergeMergedFilesetA = Fileset(name="mergeMergedFilesetA")
mergeMergedFilesetB = Fileset(name="mergeMergedFilesetB")
mergeMergedFilesetA.create()
mergeMergedFilesetB.create()
mergeWorkflow = Workflow(name="mergeWorkflow", spec="bogus",
owner="Steve", task="Test")
mergeWorkflow.create()
mergeSubscriptionA = Subscription(fileset=mergeFilesetA,
workflow=mergeWorkflow,
split_algo="WMBSMergeBySize")
mergeSubscriptionB = Subscription(fileset=mergeFilesetB,
workflow=mergeWorkflow,
split_algo="WMBSMergeBySize")
mergeSubscriptionA.create()
mergeSubscriptionB.create()
inputFileset = Fileset(name="inputFileset")
inputFileset.create()
inputFileA = File(lfn="inputLFNA")
inputFileB = File(lfn="inputLFNB")
inputFileA.create()
inputFileB.create()
procWorkflowA = Workflow(name="procWorkflowA", spec="bunk2",
owner="Steve", task="Test")
procWorkflowA.create()
procWorkflowA.addOutput("output", mergeFilesetA, mergeMergedFilesetA)
procWorkflowB = Workflow(name="procWorkflowB", spec="bunk3",
owner="Steve", task="Test2")
procWorkflowB.create()
procWorkflowB.addOutput("output", mergeFilesetB, mergeMergedFilesetB)
procSubscriptionA = Subscription(fileset=inputFileset,
workflow=procWorkflowA,
split_algo="EventBased")
procSubscriptionA.create()
procSubscriptionB = Subscription(fileset=inputFileset,
workflow=procWorkflowB,
split_algo="EventBased")
procSubscriptionB.create()
jobGroupA = JobGroup(subscription=procSubscriptionA)
jobGroupA.create()
jobGroupB = JobGroup(subscription=procSubscriptionB)
jobGroupB.create()
changeStateDAO = self.daoFactory(classname="Jobs.ChangeState")
testJobA = Job()
testJobA.addFile(inputFileA)
testJobA.create(jobGroupA)
testJobA["state"] = "cleanout"
testJobA["oldstate"] = "new"
testJobA["couch_record"] = "somejive"
testJobA["retry_count"] = 0
testJobA["outcome"] = "success"
testJobA.save()
testJobB = Job()
testJobB.addFile(inputFileB)
testJobB.create(jobGroupA)
testJobB["state"] = "cleanout"
testJobB["oldstate"] = "new"
testJobB["couch_record"] = "somejive"
testJobB["retry_count"] = 0
testJobB["outcome"] = "success"
testJobB.save()
testJobC = Job()
testJobC.addFile(inputFileA)
testJobC.create(jobGroupB)
testJobC["state"] = "cleanout"
testJobC["oldstate"] = "new"
testJobC["couch_record"] = "somejive"
testJobC["retry_count"] = 0
testJobC["outcome"] = "success"
testJobC.save()
testJobD = Job()
testJobD.addFile(inputFileA)
testJobD.create(jobGroupB)
testJobD["state"] = "cleanout"
testJobD["oldstate"] = "new"
testJobD["couch_record"] = "somejive"
testJobD["retry_count"] = 0
testJobD["outcome"] = "failure"
testJobD.save()
testJobE = Job()
testJobE.addFile(inputFileB)
testJobE.create(jobGroupB)
testJobE["state"] = "cleanout"
testJobE["oldstate"] = "new"
testJobE["couch_record"] = "somejive"
testJobE["retry_count"] = 0
testJobE["outcome"] = "success"
testJobE.save()
testJobF = Job()
testJobF.addFile(inputFileB)
testJobF.create(jobGroupB)
testJobF["state"] = "cleanout"
testJobF["oldstate"] = "new"
testJobF["couch_record"] = "somejive"
testJobF["retry_count"] = 0
testJobF["outcome"] = "failure"
testJobF.save()
changeStateDAO.execute([testJobA, testJobB, testJobC, testJobD,
testJobE, testJobF])
fileA = File(lfn="fileA", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
fileA.addRun(Run(1, *[45]))
fileA.create()
fileA.addParent(inputFileA["lfn"])
fileB = File(lfn="fileB", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
fileB.addRun(Run(1, *[45]))
fileB.create()
fileB.addParent(inputFileB["lfn"])
jobGroupA.output.addFile(fileA)
jobGroupA.output.addFile(fileB)
jobGroupA.output.commit()
mergeFilesetA.addFile(fileA)
mergeFilesetA.addFile(fileB)
mergeFilesetA.commit()
fileC = File(lfn="fileC", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
fileC.addRun(Run(1, *[45]))
fileC.create()
fileC.addParent(inputFileA["lfn"])
fileD = File(lfn="fileD", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
fileD.addRun(Run(1, *[45]))
fileD.create()
fileD.addParent(inputFileB["lfn"])
jobGroupB.output.addFile(fileC)
jobGroupB.output.addFile(fileD)
mergeFilesetB.addFile(fileC)
mergeFilesetB.addFile(fileD)
mergeFilesetB.commit()
splitter = SplitterFactory()
jobFactory = splitter(package="WMCore.WMBS",
subscription=mergeSubscriptionB)
result = jobFactory(min_merge_size=1, max_merge_size=20000,
max_merge_events=7169)
assert len(result) == 0, \
"Error: No merge jobs should have been created."
fileE = File(lfn="fileE", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
fileE.addRun(Run(1, *[45]))
fileE.create()
fileE.addParent(inputFileA["lfn"])
fileF = File(lfn="fileF", size=1024, events=1024, first_event=0,
locations={"T2_CH_CERN"})
fileF.addRun(Run(1, *[45]))
fileF.create()
fileF.addParent(inputFileB["lfn"])
jobGroupB.output.addFile(fileE)
jobGroupB.output.addFile(fileF)
mergeFilesetB.addFile(fileE)
mergeFilesetB.addFile(fileF)
mergeFilesetB.commit()
testJobD["outcome"] = "success"
testJobD.save()
testJobF["outcome"] = "success"
testJobF.save()
changeStateDAO.execute([testJobD, testJobF])
result = jobFactory(min_merge_size=1, max_merge_size=20000,
max_merge_events=7169)
assert len(result) == 1, \
"Error: One merge job should have been created: %s" % len(result)
return
def createTestJobGroup(self, name = "TestWorkthrough",
specLocation = "spec.xml", error = False,
task = "/TestWorkload/ReReco", nJobs = 10):
"""
_createTestJobGroup_
Generate a test WMBS JobGroup with real FWJRs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec = specLocation, owner = "Simon",
name = name, task = task)
testWorkflow.create()
testWMBSFileset = Fileset(name = name)
testWMBSFileset.create()
testFileA = File(lfn = makeUUID(), size = 1024, events = 10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn = makeUUID(), size = 1024, events = 10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
testSubscription = Subscription(fileset = testWMBSFileset,
workflow = testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription = testSubscription)
testJobGroup.create()
for i in range(0, nJobs):
testJob = Job(name = makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run = 10, lumis = [12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
report = Report()
if error:
path = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs", "badBackfillJobReport.pkl")
else:
path = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs", "PerformanceReport2.pkl")
report.load(filename = path)
self.changeState.propagate(testJobGroup.jobs, 'created', 'new')
self.changeState.propagate(testJobGroup.jobs, 'executing', 'created')
self.changeState.propagate(testJobGroup.jobs, 'complete', 'executing')
for job in testJobGroup.jobs:
job['fwjr'] = report
self.changeState.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
self.changeState.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed')
self.changeState.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def createTestJobGroup(self,
name="TestWorkthrough",
specLocation="spec.xml",
error=False,
task="/TestWorkload/ReReco",
nJobs=10):
"""
_createTestJobGroup_
Generate a test WMBS JobGroup with real FWJRs
"""
myThread = threading.currentThread()
testWorkflow = Workflow(spec=specLocation,
owner="Simon",
name=name,
task=task)
testWorkflow.create()
testWMBSFileset = Fileset(name=name)
testWMBSFileset.create()
testFileA = File(lfn=makeUUID(), size=1024, events=10)
testFileA.addRun(Run(10, *[12312]))
testFileA.setLocation('malpaquet')
testFileB = File(lfn=makeUUID(), size=1024, events=10)
testFileB.addRun(Run(10, *[12312]))
testFileB.setLocation('malpaquet')
testFileA.create()
testFileB.create()
testWMBSFileset.addFile(testFileA)
testWMBSFileset.addFile(testFileB)
testWMBSFileset.commit()
testWMBSFileset.markOpen(0)
testSubscription = Subscription(fileset=testWMBSFileset,
workflow=testWorkflow)
testSubscription.create()
testJobGroup = JobGroup(subscription=testSubscription)
testJobGroup.create()
for i in range(0, nJobs):
testJob = Job(name=makeUUID())
testJob.addFile(testFileA)
testJob.addFile(testFileB)
testJob['retry_count'] = 1
testJob['retry_max'] = 10
testJob['mask'].addRunAndLumis(run=10, lumis=[12312, 12313])
testJobGroup.add(testJob)
testJobGroup.commit()
report = Report()
if error:
path = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"badBackfillJobReport.pkl")
else:
path = os.path.join(WMCore.WMBase.getTestBase(),
"WMComponent_t/JobAccountant_t/fwjrs",
"PerformanceReport2.pkl")
report.load(filename=path)
self.changeState.propagate(testJobGroup.jobs, 'created', 'new')
self.changeState.propagate(testJobGroup.jobs, 'executing', 'created')
self.changeState.propagate(testJobGroup.jobs, 'complete', 'executing')
for job in testJobGroup.jobs:
job['fwjr'] = report
self.changeState.propagate(testJobGroup.jobs, 'jobfailed', 'complete')
self.changeState.propagate(testJobGroup.jobs, 'exhausted', 'jobfailed')
self.changeState.propagate(testJobGroup.jobs, 'cleanout', 'exhausted')
testSubscription.completeFiles([testFileA, testFileB])
return testJobGroup
def stuffWMBS(self, injected = True):
"""
_stuffWMBS_
Insert some dummy jobs, jobgroups, filesets, files and subscriptions
into WMBS to test job creation. Three completed job groups each
containing several files are injected. Another incomplete job group is
also injected. Also files are added to the "Mergeable" subscription as
well as to the output fileset for their jobgroups.
"""
locationAction = self.daoFactory(classname = "Locations.New")
locationAction.execute(siteName = "s1", seName = "somese.cern.ch")
changeStateDAO = self.daoFactory(classname = "Jobs.ChangeState")
self.mergeFileset = Fileset(name = "mergeFileset")
self.mergeFileset.create()
self.bogusFileset = Fileset(name = "bogusFileset")
self.bogusFileset.create()
self.mergeMergedFileset = Fileset(name = "mergeMergedFileset")
self.mergeMergedFileset.create()
self.bogusMergedFileset = Fileset(name = "bogusMergedFileset")
self.bogusMergedFileset.create()
mergeWorkflow = Workflow(name = "mergeWorkflow", spec = "bunk2",
owner = "Steve", task="Test")
mergeWorkflow.create()
markWorkflow = self.daoFactory(classname = "Workflow.MarkInjectedWorkflows")
markWorkflow.execute(names = [mergeWorkflow.name], injected = injected)
self.mergeSubscription = Subscription(fileset = self.mergeFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize")
self.mergeSubscription.create()
self.bogusSubscription = Subscription(fileset = self.bogusFileset,
workflow = mergeWorkflow,
split_algo = "WMBSMergeBySize")
inputFileset = Fileset(name = "inputFileset")
inputFileset.create()
inputWorkflow = Workflow(name = "inputWorkflow", spec = "input",
owner = "Steve", task = "Test")
inputWorkflow.create()
inputWorkflow.addOutput("output", self.mergeFileset,
self.mergeMergedFileset)
inputWorkflow.addOutput("output2", self.bogusFileset,
self.bogusMergedFileset)
bogusInputWorkflow = Workflow(name = "bogusInputWorkflow", spec = "input",
owner = "Steve", task = "Test")
bogusInputWorkflow.create()
inputSubscription = Subscription(fileset = inputFileset,
workflow = inputWorkflow)
inputSubscription.create()
bogusInputSubscription = Subscription(fileset = inputFileset,
workflow = bogusInputWorkflow)
bogusInputSubscription.create()
parentFile1 = File(lfn = "parentFile1")
parentFile1.create()
parentFile2 = File(lfn = "parentFile2")
parentFile2.create()
parentFile3 = File(lfn = "parentFile3")
parentFile3.create()
parentFile4 = File(lfn = "parentFile4")
parentFile4.create()
self.parentFileSite2 = File(lfn = "parentFileSite2")
self.parentFileSite2.create()
jobGroup1 = JobGroup(subscription = inputSubscription)
jobGroup1.create()
jobGroup2 = JobGroup(subscription = inputSubscription)
jobGroup2.create()
jobGroup3 = JobGroup(subscription = bogusInputSubscription)
jobGroup3.create()
testJob1 = Job()
testJob1.addFile(parentFile1)
testJob1.create(jobGroup1)
testJob1["state"] = "cleanout"
testJob1["oldstate"] = "new"
testJob1["couch_record"] = "somejive"
testJob1["retry_count"] = 0
testJob1["outcome"] = "success"
testJob1.save()
changeStateDAO.execute([testJob1])
testJob1A = Job()
testJob1A.addFile(parentFile1)
testJob1A.create(jobGroup3)
testJob1A["state"] = "cleanout"
testJob1A["oldstate"] = "new"
testJob1A["couch_record"] = "somejive"
testJob1A["retry_count"] = 0
testJob1A["outcome"] = "failure"
testJob1A.save()
changeStateDAO.execute([testJob1A])
testJob2 = Job()
testJob2.addFile(parentFile2)
testJob2.create(jobGroup1)
testJob2["state"] = "cleanout"
testJob2["oldstate"] = "new"
testJob2["couch_record"] = "somejive"
testJob2["retry_count"] = 0
testJob2["outcome"] = "success"
testJob2.save()
changeStateDAO.execute([testJob2])
testJob3 = Job()
testJob3.addFile(parentFile3)
testJob3.create(jobGroup2)
testJob3["state"] = "cleanout"
testJob3["oldstate"] = "new"
testJob3["couch_record"] = "somejive"
testJob3["retry_count"] = 0
testJob3["outcome"] = "success"
testJob3.save()
changeStateDAO.execute([testJob3])
testJob4 = Job()
testJob4.addFile(parentFile4)
testJob4.create(jobGroup2)
testJob4["state"] = "cleanout"
testJob4["oldstate"] = "new"
testJob4["couch_record"] = "somejive"
testJob4["retry_count"] = 0
testJob4["outcome"] = "failure"
testJob4.save()
changeStateDAO.execute([testJob4])
# We'll simulate a failed split by event job that the merger should
# ignore.
parentFile5 = File(lfn = "parentFile5")
parentFile5.create()
testJob5 = Job()
testJob5.addFile(parentFile5)
testJob5.create(jobGroup2)
testJob5["state"] = "cleanout"
testJob5["oldstate"] = "new"
testJob5["couch_record"] = "somejive"
testJob5["retry_count"] = 0
testJob5["outcome"] = "success"
testJob5.save()
changeStateDAO.execute([testJob5])
testJob6 = Job()
testJob6.addFile(parentFile5)
testJob6.create(jobGroup2)
testJob6["state"] = "cleanout"
testJob6["oldstate"] = "new"
testJob6["couch_record"] = "somejive"
testJob6["retry_count"] = 0
testJob6["outcome"] = "failure"
testJob6.save()
changeStateDAO.execute([testJob6])
testJob7 = Job()
testJob7.addFile(self.parentFileSite2)
testJob7.create(jobGroup2)
testJob7["state"] = "cleanout"
testJob7["oldstate"] = "new"
testJob7["couch_record"] = "somejive"
testJob7["retry_count"] = 0
testJob7["outcome"] = "success"
testJob7.save()
changeStateDAO.execute([testJob7])
badFile1 = File(lfn = "badFile1", size = 10241024, events = 10241024,
first_event = 0, locations = set(["somese.cern.ch"]))
badFile1.addRun(Run(1, *[45]))
badFile1.create()
badFile1.addParent(parentFile5["lfn"])
file1 = File(lfn = "file1", size = 1024, events = 1024, first_event = 0,
locations = set(["somese.cern.ch"]))
file1.addRun(Run(1, *[45]))
file1.create()
file1.addParent(parentFile1["lfn"])
file2 = File(lfn = "file2", size = 1024, events = 1024,
first_event = 1024, locations = set(["somese.cern.ch"]))
file2.addRun(Run(1, *[45]))
file2.create()
file2.addParent(parentFile1["lfn"])
file3 = File(lfn = "file3", size = 1024, events = 1024,
first_event = 2048, locations = set(["somese.cern.ch"]))
file3.addRun(Run(1, *[45]))
file3.create()
file3.addParent(parentFile1["lfn"])
file4 = File(lfn = "file4", size = 1024, events = 1024,
first_event = 3072, locations = set(["somese.cern.ch"]))
file4.addRun(Run(1, *[45]))
file4.create()
file4.addParent(parentFile1["lfn"])
fileA = File(lfn = "fileA", size = 1024, events = 1024,
first_event = 0, locations = set(["somese.cern.ch"]))
fileA.addRun(Run(1, *[46]))
fileA.create()
fileA.addParent(parentFile2["lfn"])
fileB = File(lfn = "fileB", size = 1024, events = 1024,
first_event = 1024, locations = set(["somese.cern.ch"]))
fileB.addRun(Run(1, *[46]))
fileB.create()
fileB.addParent(parentFile2["lfn"])
fileC = File(lfn = "fileC", size = 1024, events = 1024,
first_event = 2048, locations = set(["somese.cern.ch"]))
fileC.addRun(Run(1, *[46]))
fileC.create()
fileC.addParent(parentFile2["lfn"])
fileI = File(lfn = "fileI", size = 1024, events = 1024,
first_event = 0, locations = set(["somese.cern.ch"]))
fileI.addRun(Run(2, *[46]))
fileI.create()
fileI.addParent(parentFile3["lfn"])
fileII = File(lfn = "fileII", size = 1024, events = 1024,
first_event = 1024, locations = set(["somese.cern.ch"]))
fileII.addRun(Run(2, *[46]))
fileII.create()
fileII.addParent(parentFile3["lfn"])
fileIII = File(lfn = "fileIII", size = 1024, events = 1024,
first_event = 2048, locations = set(["somese.cern.ch"]))
fileIII.addRun(Run(2, *[46]))
fileIII.create()
fileIII.addParent(parentFile3["lfn"])
fileIV = File(lfn = "fileIV", size = 1024, events = 1024,
first_event = 3072, locations = set(["somese.cern.ch"]))
fileIV.addRun(Run(2, *[46]))
fileIV.create()
fileIV.addParent(parentFile3["lfn"])
fileX = File(lfn = "badFileA", size = 1024, events = 1024,
first_event = 0, locations = set(["somese.cern.ch"]))
fileX.addRun(Run(1, *[47]))
fileX.create()
fileX.addParent(parentFile4["lfn"])
fileY = File(lfn = "badFileB", size = 1024, events = 1024,
first_event = 1024, locations = set(["somese.cern.ch"]))
fileY.addRun(Run(1, *[47]))
fileY.create()
fileY.addParent(parentFile4["lfn"])
fileZ = File(lfn = "badFileC", size = 1024, events = 1024,
first_event = 2048, locations = set(["somese.cern.ch"]))
fileZ.addRun(Run(1, *[47]))
fileZ.create()
fileZ.addParent(parentFile4["lfn"])
jobGroup1.output.addFile(file1)
jobGroup1.output.addFile(file2)
jobGroup1.output.addFile(file3)
jobGroup1.output.addFile(file4)
jobGroup1.output.addFile(fileA)
jobGroup1.output.addFile(fileB)
jobGroup1.output.addFile(fileC)
jobGroup1.output.commit()
jobGroup2.output.addFile(fileI)
jobGroup2.output.addFile(fileII)
jobGroup2.output.addFile(fileIII)
jobGroup2.output.addFile(fileIV)
jobGroup2.output.addFile(fileX)
jobGroup2.output.addFile(fileY)
jobGroup2.output.addFile(fileZ)
jobGroup2.output.addFile(badFile1)
jobGroup2.output.commit()
for file in [file1, file2, file3, file4, fileA, fileB, fileC, fileI,
fileII, fileIII, fileIV, fileX, fileY, fileZ, badFile1]:
self.mergeFileset.addFile(file)
self.bogusFileset.addFile(file)
self.mergeFileset.commit()
self.bogusFileset.commit()
return
def makeNJobs(self,
name,
task,
nJobs,
jobGroup,
fileset,
sub,
site=None,
bl=[],
wl=[]):
"""
_makeNJobs_
Make and return a WMBS Job and File
This handles all those damn add-ons
"""
# Set the CacheDir
cacheDir = os.path.join(self.testDir, 'CacheDir')
for n in range(nJobs):
# First make a file
# site = self.sites[0]
testFile = File(lfn="/singleLfn/%s/%s" % (name, n),
size=1024,
events=10)
if site:
testFile.setLocation(site)
else:
for tmpSite in self.sites:
testFile.setLocation('se.%s' % (tmpSite))
testFile.create()
fileset.addFile(testFile)
fileset.commit()
index = 0
for f in fileset.files:
index += 1
testJob = Job(name='%s-%i' % (name, index))
testJob.addFile(f)
testJob["location"] = f.getLocations()[0]
testJob['custom']['location'] = f.getLocations()[0]
testJob['task'] = task.getPathName()
testJob['sandbox'] = task.data.input.sandbox
testJob['spec'] = os.path.join(self.testDir, 'basicWorkload.pcl')
testJob['mask']['FirstEvent'] = 101
testJob['owner'] = 'tapas'
testJob["siteBlacklist"] = bl
testJob["siteWhitelist"] = wl
testJob['ownerDN'] = 'tapas'
testJob['ownerRole'] = 'cmsrole'
testJob['ownerGroup'] = 'phgroup'
jobCache = os.path.join(cacheDir, 'Sub_%i' % (sub),
'Job_%i' % (index))
os.makedirs(jobCache)
testJob.create(jobGroup)
testJob['cache_dir'] = jobCache
testJob.save()
jobGroup.add(testJob)
with open(os.path.join(jobCache, 'job.pkl'), 'wb') as output:
pickle.dump(testJob, output)
return testJob, testFile
