def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose = False):
"""
__createSubscriptionsInWMBS_
Create subscriptions in WMBS for all the tasks in the spec. This
includes filesets, workflows and the output map for each task.
"""
# create runtime sandbox for workflow
self.createSandbox()
#FIXME: Let workflow put in values if spec is missing them
workflow = Workflow(spec = self.wmSpec.specUrl(), owner = self.wmSpec.getOwner()["name"],
dn = self.wmSpec.getOwner().get("dn", "unknown"),
group = self.wmSpec.getOwner().get("group", "unknown"),
owner_vogroup = self.wmSpec.getOwner().get("vogroup", "DEFAULT"),
owner_vorole = self.wmSpec.getOwner().get("vorole", "DEFAULT"),
name = self.wmSpec.name(), task = task.getPathName(),
wfType = self.wmSpec.getDashboardActivity(),
alternativeFilesetClose = alternativeFilesetClose,
priority = self.wmSpec.priority())
workflow.create()
subscription = Subscription(fileset = fileset, workflow = workflow,
split_algo = task.jobSplittingAlgorithm(),
type = task.getPrimarySubType())
if subscription.exists():
subscription.load()
msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)"
self.logger.info(msg % (subscription['id'], task.getPathName()))
else:
subscription.create()
for site in task.siteWhitelist():
subscription.addWhiteBlackList([{"site_name": site, "valid": True}])
for site in task.siteBlacklist():
subscription.addWhiteBlackList([{"site_name": site, "valid": False}])
if self.topLevelSubscription == None:
self.topLevelSubscription = subscription
logging.info("Top level subscription created: %s" % subscription["id"])
else:
logging.info("Child subscription created: %s" % subscription["id"])
outputModules = task.getOutputModulesForTask()
for outputModule in outputModules:
for outputModuleName in outputModule.listSections_():
outputFileset = Fileset(self.outputFilesetName(task, outputModuleName))
outputFileset.create()
outputFileset.markOpen(True)
mergedOutputFileset = None
for childTask in task.childTaskIterator():
if childTask.data.input.outputModule == outputModuleName:
if childTask.taskType() == "Merge":
mergedOutputFileset = Fileset(self.outputFilesetName(childTask, "Merged"))
mergedOutputFileset.create()
mergedOutputFileset.markOpen(True)
primaryDataset = getattr(getattr(outputModule, outputModuleName), "primaryDataset", None)
if primaryDataset != None:
self.mergeOutputMapping[mergedOutputFileset.id] = primaryDataset
self._createSubscriptionsInWMBS(childTask, outputFileset, alternativeFilesetClose)
if mergedOutputFileset == None:
workflow.addOutput(outputModuleName, outputFileset,
outputFileset)
else:
workflow.addOutput(outputModuleName, outputFileset,
mergedOutputFileset)
return self.topLevelSubscription
def createJobs(self):
"""
_createJobs_
Create test jobs in WMBS and BossAir
"""
testWorkflow = Workflow(spec=makeUUID(), owner="tapas",
name=makeUUID(), task="Test")
testWorkflow.create()
testFilesetA = Fileset(name="TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name="TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name="TestFilesetC")
testFilesetC.create()
testFileA = File(lfn="testFileA", locations=set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset=testFilesetA,
workflow=testWorkflow,
type="Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{"site_name": "testSite1", "valid": True}])
testSubscriptionB = Subscription(fileset=testFilesetB,
workflow=testWorkflow,
type="Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{"site_name": "testSite1", "valid": False}])
testSubscriptionC = Subscription(fileset=testFilesetC,
workflow=testWorkflow,
type="Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription=testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription=testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription=testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name="testJobA", files=[testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group=testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name="testJobB", files=[testFileA])
testJobB["couch_record"] = makeUUID()
testJobB["cache_dir"] = self.tempDir
testJobB.create(group=testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name="testJobC", files=[testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group=testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name="testJobD", files=[testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group=testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name="testJobE", files=[testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group=testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name="testJobF", files=[testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group=testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name="testJobG", files=[testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group=testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name="testJobH", files=[testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group=testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name="testJobI", files=[testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group=testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name="testJobJ", files=[testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group=testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
changeStateAction.execute(jobs=[testJobA, testJobB, testJobC, testJobD,
testJobE, testJobF, testJobG, testJobH,
testJobI, testJobJ])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite2")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
return
def testList(self):
"""
_testList_
Test the functions that list thresholds for creating jobs and submitting
jobs.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1",
"T1_US_FNAL", "LsfPlugin")
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)
testWorkflow = Workflow(spec=makeUUID(),
owner="Steve",
name=makeUUID(),
task="Test")
testWorkflow.create()
testFilesetA = Fileset(name="TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name="TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name="TestFilesetC")
testFilesetC.create()
testFileA = File(lfn="testFileA",
locations=set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset=testFilesetA,
workflow=testWorkflow,
type="Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{
"site_name": "testSite1",
"valid": True
}])
testSubscriptionB = Subscription(fileset=testFilesetB,
workflow=testWorkflow,
type="Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{
"site_name": "testSite1",
"valid": False
}])
testSubscriptionC = Subscription(fileset=testFilesetC,
workflow=testWorkflow,
type="Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription=testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription=testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription=testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name="testJobA", files=[testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group=testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name="testJobB", files=[testFileA])
testJobB["couch_record"] = makeUUID()
testJobB.create(group=testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name="testJobC", files=[testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group=testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name="testJobD", files=[testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group=testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name="testJobE", files=[testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group=testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name="testJobF", files=[testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group=testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name="testJobG", files=[testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group=testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name="testJobH", files=[testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group=testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name="testJobI", files=[testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group=testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name="testJobJ", files=[testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group=testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname="Jobs.ChangeState")
changeStateAction.execute(jobs=[
testJobA, testJobB, testJobC, testJobD, testJobE, testJobF,
testJobG, testJobH, testJobI, testJobJ
])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname="Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite1")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
createThresholds = myResourceControl.listThresholdsForCreate()
submitThresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual(len(createThresholds.keys()), 2,
"Error: Wrong number of sites in create thresholds")
self.assertEqual(createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of slots for site 1")
self.assertEqual(createThresholds["testSite2"]["total_slots"], 20,
"Error: Wrong number of slots for site 2")
# We should have two running jobs with locations at site one,
# two running jobs without locations at site two, and one running
# job without a location at site one and two.
self.assertEqual(createThresholds["testSite1"]["pending_jobs"], 4,
"Error: Wrong number of pending jobs for site 1")
# We should have one running job with a location at site 2 and
# another running job without a location.
self.assertEqual(createThresholds["testSite2"]["pending_jobs"], 2,
"Error: Wrong number of pending jobs for site 2")
# We should also have a phedex_name
self.assertEqual(createThresholds["testSite1"]["cms_name"],
"T1_US_FNAL")
self.assertEqual(createThresholds["testSite2"]["cms_name"], None)
mergeThreshold1 = None
mergeThreshold2 = None
procThreshold1 = None
procThreshold2 = None
self.assertEqual(submitThresholds["testSite1"]['cms_name'],
'T1_US_FNAL')
for threshold in submitThresholds["testSite1"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold1 = threshold
elif threshold['task_type'] == "Processing":
procThreshold1 = threshold
self.assertEqual(submitThresholds["testSite2"]['cms_name'], None)
for threshold in submitThresholds["testSite2"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold2 = threshold
elif threshold['task_type'] == "Processing":
procThreshold2 = threshold
self.assertEqual(
submitThresholds["testSite1"]["total_running_jobs"], 1,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite2"]["total_running_jobs"], 0,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite1"]["total_pending_jobs"], 1,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(
submitThresholds["testSite2"]["total_pending_jobs"], 0,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(
mergeThreshold1["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold1["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold1["task_running_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold1["task_pending_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
mergeThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(
procThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
return
def _createSubscriptionsInWMBS(self,
task,
fileset,
alternativeFilesetClose=False):
"""
__createSubscriptionsInWMBS_
Create subscriptions in WMBS for all the tasks in the spec. This
includes filesets, workflows and the output map for each task.
"""
# create runtime sandbox for workflow
self.createSandbox()
# FIXME: Let workflow put in values if spec is missing them
workflow = Workflow(
spec=self.wmSpec.specUrl(),
owner=self.wmSpec.getOwner()["name"],
dn=self.wmSpec.getOwner().get("dn", "unknown"),
group=self.wmSpec.getOwner().get("group", "unknown"),
owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"),
owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"),
name=self.wmSpec.name(),
task=task.getPathName(),
wfType=self.wmSpec.getDashboardActivity(),
alternativeFilesetClose=alternativeFilesetClose,
priority=self.wmSpec.priority())
workflow.create()
subscription = Subscription(fileset=fileset,
workflow=workflow,
split_algo=task.jobSplittingAlgorithm(),
type=task.getPrimarySubType())
if subscription.exists():
subscription.load()
msg = "Subscription %s already exists for %s (you may ignore file insertion messages below, existing files wont be duplicated)"
self.logger.info(msg % (subscription['id'], task.getPathName()))
else:
subscription.create()
for site in task.siteWhitelist():
subscription.addWhiteBlackList([{
"site_name": site,
"valid": True
}])
for site in task.siteBlacklist():
subscription.addWhiteBlackList([{
"site_name": site,
"valid": False
}])
if self.topLevelSubscription is None:
self.topLevelSubscription = subscription
logging.info("Top level subscription created: %s",
subscription["id"])
else:
logging.info("Child subscription created: %s", subscription["id"])
outputModules = task.getOutputModulesForTask()
ignoredOutputModules = task.getIgnoredOutputModulesForTask()
for outputModule in outputModules:
for outputModuleName in outputModule.listSections_():
if outputModuleName in ignoredOutputModules:
logging.info(
"IgnoredOutputModule set for %s, skipping fileset creation.",
outputModuleName)
continue
outputFileset = Fileset(
self.outputFilesetName(task, outputModuleName))
outputFileset.create()
outputFileset.markOpen(True)
mergedOutputFileset = None
for childTask in task.childTaskIterator():
if childTask.data.input.outputModule == outputModuleName:
if childTask.taskType() == "Merge":
mergedOutputFileset = Fileset(
self.outputFilesetName(childTask, "Merged"))
mergedOutputFileset.create()
mergedOutputFileset.markOpen(True)
primaryDataset = getattr(
getattr(outputModule, outputModuleName),
"primaryDataset", None)
if primaryDataset != None:
self.mergeOutputMapping[
mergedOutputFileset.id] = primaryDataset
self._createSubscriptionsInWMBS(
childTask, outputFileset, alternativeFilesetClose)
if mergedOutputFileset is None:
workflow.addOutput(outputModuleName, outputFileset,
outputFileset)
else:
workflow.addOutput(outputModuleName, outputFileset,
mergedOutputFileset)
return self.topLevelSubscription
def createJobs(self):
"""
_createJobs_
Create test jobs in WMBS and BossAir
"""
testWorkflow = Workflow(spec = makeUUID(), owner = "tapas",
name = makeUUID(), task = "Test")
testWorkflow.create()
testFilesetA = Fileset(name = "TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name = "TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name = "TestFilesetC")
testFilesetC.create()
testFileA = File(lfn = "testFileA", locations = set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset = testFilesetA,
workflow = testWorkflow,
type = "Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{"site_name": "testSite1", "valid": True}])
testSubscriptionB = Subscription(fileset = testFilesetB,
workflow = testWorkflow,
type = "Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{"site_name": "testSite1", "valid": False}])
testSubscriptionC = Subscription(fileset = testFilesetC,
workflow = testWorkflow,
type = "Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription = testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription = testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription = testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name = "testJobA", files = [testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group = testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name = "testJobB", files = [testFileA])
testJobB["couch_record"] = makeUUID()
testJobB["cache_dir"] = self.tempDir
testJobB.create(group = testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name = "testJobC", files = [testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group = testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name = "testJobD", files = [testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group = testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name = "testJobE", files = [testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group = testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name = "testJobF", files = [testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group = testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name = "testJobG", files = [testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group = testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name = "testJobH", files = [testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group = testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name = "testJobI", files = [testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group = testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name = "testJobJ", files = [testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group = testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname = "Jobs.ChangeState")
changeStateAction.execute(jobs = [testJobA, testJobB, testJobC, testJobD,
testJobE, testJobF, testJobG, testJobH,
testJobI, testJobJ])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname = "Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite2")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
return
def _createSubscriptionsInWMBS(self,
task,
fileset,
alternativeFilesetClose=False):
"""
__createSubscriptionsInWMBS_
Create subscriptions in WMBS for all the tasks in the spec. This
includes filesets, workflows and the output map for each task.
"""
# create runtime sandbox for workflow
self.createSandbox()
# FIXME: Let workflow put in values if spec is missing them
workflow = Workflow(
spec=self.wmSpec.specUrl(),
owner=self.wmSpec.getOwner()["name"],
dn=self.wmSpec.getOwner().get("dn", "unknown"),
group=self.wmSpec.getOwner().get("group", "unknown"),
owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"),
owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"),
name=self.wmSpec.name(),
task=task.getPathName(),
wfType=self.wmSpec.getDashboardActivity(),
alternativeFilesetClose=alternativeFilesetClose,
priority=self.wmSpec.priority())
workflow.create()
subscription = Subscription(fileset=fileset,
workflow=workflow,
split_algo=task.jobSplittingAlgorithm(),
type=task.getPrimarySubType())
subscription.create()
### FIXME: I'm pretty sure we can improve how we handle this site white/black list
for site in task.siteWhitelist():
subscription.addWhiteBlackList([{
"site_name": site,
"valid": True
}])
for site in task.siteBlacklist():
subscription.addWhiteBlackList([{
"site_name": site,
"valid": False
}])
if self.topLevelSubscription is None:
self.topLevelSubscription = subscription
logging.info("Top level subscription %s created for %s",
subscription["id"], self.wmSpec.name())
else:
logging.info("Child subscription %s created for %s",
subscription["id"], self.wmSpec.name())
outputModules = task.getOutputModulesForTask()
ignoredOutputModules = task.getIgnoredOutputModulesForTask()
for outputModule in outputModules:
for outputModuleName in outputModule.listSections_():
if outputModuleName in ignoredOutputModules:
msg = "%s has %s as IgnoredOutputModule, skipping fileset creation."
logging.info(msg, task.getPathName(), outputModuleName)
continue
dataTier = getattr(getattr(outputModule, outputModuleName),
"dataTier", '')
filesetName = self.outputFilesetName(task, outputModuleName,
dataTier)
outputFileset = Fileset(filesetName)
outputFileset.create()
outputFileset.markOpen(True)
mergedOutputFileset = None
for childTask in task.childTaskIterator():
if childTask.data.input.outputModule == outputModuleName:
childDatatier = getattr(childTask.data.input,
'dataTier', '')
if childTask.taskType() in [
"Cleanup", "Merge"
] and childDatatier != dataTier:
continue
elif childTask.taskType(
) == "Merge" and childDatatier == dataTier:
filesetName = self.outputFilesetName(
childTask, "Merged", dataTier)
mergedOutputFileset = Fileset(filesetName)
mergedOutputFileset.create()
mergedOutputFileset.markOpen(True)
primaryDataset = getattr(
getattr(outputModule, outputModuleName),
"primaryDataset", None)
if primaryDataset is not None:
self.mergeOutputMapping[
mergedOutputFileset.id] = primaryDataset
self._createSubscriptionsInWMBS(
childTask, outputFileset, alternativeFilesetClose)
if mergedOutputFileset is None:
workflow.addOutput(outputModuleName + dataTier,
outputFileset, outputFileset)
else:
workflow.addOutput(outputModuleName + dataTier,
outputFileset, mergedOutputFileset)
return
def testList(self):
"""
_testList_
Test the functions that list thresholds for creating jobs and submitting
jobs.
"""
myResourceControl = ResourceControl()
myResourceControl.insertSite("testSite1", 10, 20, "testSE1", "testCE1", "T1_US_FNAL", "LsfPlugin")
myResourceControl.insertSite("testSite2", 20, 40, "testSE2", "testCE2")
myResourceControl.insertThreshold("testSite1", "Processing", 20, 10)
myResourceControl.insertThreshold("testSite1", "Merge", 200, 100)
myResourceControl.insertThreshold("testSite2", "Processing", 50, 25)
myResourceControl.insertThreshold("testSite2", "Merge", 135, 65)
testWorkflow = Workflow(spec = makeUUID(), owner = "Steve",
name = makeUUID(), task = "Test")
testWorkflow.create()
testFilesetA = Fileset(name = "TestFilesetA")
testFilesetA.create()
testFilesetB = Fileset(name = "TestFilesetB")
testFilesetB.create()
testFilesetC = Fileset(name = "TestFilesetC")
testFilesetC.create()
testFileA = File(lfn = "testFileA", locations = set(["testSE1", "testSE2"]))
testFileA.create()
testFilesetA.addFile(testFileA)
testFilesetA.commit()
testFilesetB.addFile(testFileA)
testFilesetB.commit()
testFilesetC.addFile(testFileA)
testFilesetC.commit()
testSubscriptionA = Subscription(fileset = testFilesetA,
workflow = testWorkflow,
type = "Processing")
testSubscriptionA.create()
testSubscriptionA.addWhiteBlackList([{"site_name": "testSite1", "valid": True}])
testSubscriptionB = Subscription(fileset = testFilesetB,
workflow = testWorkflow,
type = "Processing")
testSubscriptionB.create()
testSubscriptionB.addWhiteBlackList([{"site_name": "testSite1", "valid": False}])
testSubscriptionC = Subscription(fileset = testFilesetC,
workflow = testWorkflow,
type = "Merge")
testSubscriptionC.create()
testJobGroupA = JobGroup(subscription = testSubscriptionA)
testJobGroupA.create()
testJobGroupB = JobGroup(subscription = testSubscriptionB)
testJobGroupB.create()
testJobGroupC = JobGroup(subscription = testSubscriptionC)
testJobGroupC.create()
# Site1, Has been assigned a location and is complete.
testJobA = Job(name = "testJobA", files = [testFileA])
testJobA["couch_record"] = makeUUID()
testJobA.create(group = testJobGroupA)
testJobA["state"] = "success"
# Site 1, Has been assigned a location and is incomplete.
testJobB = Job(name = "testJobB", files = [testFileA])
testJobB["couch_record"] = makeUUID()
testJobB.create(group = testJobGroupA)
testJobB["state"] = "executing"
runJobB = RunJob()
runJobB.buildFromJob(testJobB)
runJobB["status"] = "PEND"
# Does not have a location, white listed to site 1
testJobC = Job(name = "testJobC", files = [testFileA])
testJobC["couch_record"] = makeUUID()
testJobC.create(group = testJobGroupA)
testJobC["state"] = "new"
# Site 2, Has been assigned a location and is complete.
testJobD = Job(name = "testJobD", files = [testFileA])
testJobD["couch_record"] = makeUUID()
testJobD.create(group = testJobGroupB)
testJobD["state"] = "success"
# Site 2, Has been assigned a location and is incomplete.
testJobE = Job(name = "testJobE", files = [testFileA])
testJobE["couch_record"] = makeUUID()
testJobE.create(group = testJobGroupB)
testJobE["state"] = "executing"
runJobE = RunJob()
runJobE.buildFromJob(testJobE)
runJobE["status"] = "RUN"
# Does not have a location, site 1 is blacklisted.
testJobF = Job(name = "testJobF", files = [testFileA])
testJobF["couch_record"] = makeUUID()
testJobF.create(group = testJobGroupB)
testJobF["state"] = "new"
# Site 3, Has been assigned a location and is complete.
testJobG = Job(name = "testJobG", files = [testFileA])
testJobG["couch_record"] = makeUUID()
testJobG.create(group = testJobGroupC)
testJobG["state"] = "cleanout"
# Site 3, Has been assigned a location and is incomplete.
testJobH = Job(name = "testJobH", files = [testFileA])
testJobH["couch_record"] = makeUUID()
testJobH.create(group = testJobGroupC)
testJobH["state"] = "new"
# Site 3, Does not have a location.
testJobI = Job(name = "testJobI", files = [testFileA])
testJobI["couch_record"] = makeUUID()
testJobI.create(group = testJobGroupC)
testJobI["state"] = "new"
# Site 3, Does not have a location and is in cleanout.
testJobJ = Job(name = "testJobJ", files = [testFileA])
testJobJ["couch_record"] = makeUUID()
testJobJ.create(group = testJobGroupC)
testJobJ["state"] = "cleanout"
changeStateAction = self.daoFactory(classname = "Jobs.ChangeState")
changeStateAction.execute(jobs = [testJobA, testJobB, testJobC, testJobD,
testJobE, testJobF, testJobG, testJobH,
testJobI, testJobJ])
self.insertRunJob.execute([runJobB, runJobE])
setLocationAction = self.daoFactory(classname = "Jobs.SetLocation")
setLocationAction.execute(testJobA["id"], "testSite1")
setLocationAction.execute(testJobB["id"], "testSite1")
setLocationAction.execute(testJobD["id"], "testSite1")
setLocationAction.execute(testJobE["id"], "testSite1")
setLocationAction.execute(testJobG["id"], "testSite1")
setLocationAction.execute(testJobH["id"], "testSite1")
createThresholds = myResourceControl.listThresholdsForCreate()
submitThresholds = myResourceControl.listThresholdsForSubmit()
self.assertEqual( len(createThresholds.keys()), 2,
"Error: Wrong number of sites in create thresholds" )
self.assertEqual( createThresholds["testSite1"]["total_slots"], 10,
"Error: Wrong number of slots for site 1" )
self.assertEqual( createThresholds["testSite2"]["total_slots"], 20,
"Error: Wrong number of slots for site 2" )
# We should have two running jobs with locations at site one,
# two running jobs without locations at site two, and one running
# job without a location at site one and two.
self.assertEqual( createThresholds["testSite1"]["pending_jobs"], 4,
"Error: Wrong number of pending jobs for site 1" )
# We should have one running job with a location at site 2 and
# another running job without a location.
self.assertEqual( createThresholds["testSite2"]["pending_jobs"], 2,
"Error: Wrong number of pending jobs for site 2" )
# We should also have a phedex_name
self.assertEqual(createThresholds["testSite1"]["cms_name"], "T1_US_FNAL")
self.assertEqual(createThresholds["testSite2"]["cms_name"], None)
mergeThreshold1 = None
mergeThreshold2 = None
procThreshold1 = None
procThreshold2 = None
self.assertEqual(submitThresholds["testSite1"]['cms_name'], 'T1_US_FNAL')
for threshold in submitThresholds["testSite1"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold1 = threshold
elif threshold['task_type'] == "Processing":
procThreshold1 = threshold
self.assertEqual(submitThresholds["testSite2"]['cms_name'], None)
for threshold in submitThresholds["testSite2"]["thresholds"]:
if threshold['task_type'] == "Merge":
mergeThreshold2 = threshold
elif threshold['task_type'] == "Processing":
procThreshold2 = threshold
self.assertEqual(submitThresholds["testSite1"]["total_running_jobs"], 1,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(submitThresholds["testSite2"]["total_running_jobs"], 0,
"Error: Wrong number of running jobs for submit thresholds.")
self.assertEqual(submitThresholds["testSite1"]["total_pending_jobs"], 1,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(submitThresholds["testSite2"]["total_pending_jobs"], 0,
"Error: Wrong number of pending jobs for submit thresholds.")
self.assertEqual(mergeThreshold1["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(mergeThreshold1["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold1["task_running_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold1["task_pending_jobs"], 1,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(mergeThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(mergeThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold2["task_running_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
self.assertEqual(procThreshold2["task_pending_jobs"], 0,
"Error: Wrong number of task running jobs for submit thresholds.")
return
def createSubscription(self, topLevelFilesetName = None, task = None,
fileset = None):
"""
_createSubscription_
Create subscriptions in WMBS for all the tasks in the spec. This
includes filesets, workflows and the output map for each task.
"""
if task == None or fileset == None:
self.createTopLevelFileset(topLevelFilesetName)
sub = None
for topLevelTask in self.wmSpec.getTopLevelTask():
sub = self.createSubscription(topLevelFilesetName,
topLevelTask,
self.topLevelFileset)
return sub
# create runtime sandbox for workflow
self.createSandbox()
workflow = Workflow(spec = self.wmSpec.specUrl(), owner = self.wmSpec.getOwner()["name"],
dn = self.wmSpec.getOwner().get("dn", None),
group = self.wmSpec.getOwner().get("group", None),
owner_vogroup = self.wmSpec.getOwner().get("vogroup", ''),
owner_vorole = self.wmSpec.getOwner().get("vorole", ''),
name = self.wmSpec.name(), task = task.getPathName(),
wfType = self.wmSpec.getDashboardActivity())
workflow.create()
subscription = Subscription(fileset = fileset, workflow = workflow,
split_algo = task.jobSplittingAlgorithm(),
type = task.taskType())
subscription.create()
for site in task.siteWhitelist():
subscription.addWhiteBlackList([{"site_name": site, "valid": True}])
for site in task.siteBlacklist():
subscription.addWhiteBlackList([{"site_name": site, "valid": False}])
if self.topLevelSubscription == None:
self.topLevelSubscription = subscription
logging.info("Top level subscription created: %s" % subscription["id"])
else:
logging.info("Child subscription created: %s" % subscription["id"])
outputModules = task.getOutputModulesForTask()
for outputModule in outputModules:
for outputModuleName in outputModule.listSections_():
outputFileset = Fileset(self.outputFilesetName(task, outputModuleName))
outputFileset.create()
outputFileset.markOpen(True)
mergedOutputFileset = None
for childTask in task.childTaskIterator():
if childTask.data.input.outputModule == outputModuleName:
if childTask.taskType() == "Merge":
mergedOutputFileset = Fileset(self.outputFilesetName(childTask, "Merged"))
mergedOutputFileset.create()
mergedOutputFileset.markOpen(True)
self.createSubscription(topLevelFilesetName, childTask, outputFileset)
if mergedOutputFileset == None:
workflow.addOutput(outputModuleName, outputFileset,
outputFileset)
else:
workflow.addOutput(outputModuleName, outputFileset,
mergedOutputFileset)
return self.topLevelSubscription
def _createSubscriptionsInWMBS(self, task, fileset, alternativeFilesetClose=False):
"""
__createSubscriptionsInWMBS_
Create subscriptions in WMBS for all the tasks in the spec. This
includes filesets, workflows and the output map for each task.
"""
# create runtime sandbox for workflow
self.createSandbox()
# FIXME: Let workflow put in values if spec is missing them
workflow = Workflow(spec=self.wmSpec.specUrl(), owner=self.wmSpec.getOwner()["name"],
dn=self.wmSpec.getOwner().get("dn", "unknown"),
group=self.wmSpec.getOwner().get("group", "unknown"),
owner_vogroup=self.wmSpec.getOwner().get("vogroup", "DEFAULT"),
owner_vorole=self.wmSpec.getOwner().get("vorole", "DEFAULT"),
name=self.wmSpec.name(), task=task.getPathName(),
wfType=self.wmSpec.getDashboardActivity(),
alternativeFilesetClose=alternativeFilesetClose,
priority=self.wmSpec.priority())
workflow.create()
subscription = Subscription(fileset=fileset, workflow=workflow,
split_algo=task.jobSplittingAlgorithm(),
type=task.getPrimarySubType())
subscription.create()
### FIXME: I'm pretty sure we can improve how we handle this site white/black list
for site in task.siteWhitelist():
subscription.addWhiteBlackList([{"site_name": site, "valid": True}])
for site in task.siteBlacklist():
subscription.addWhiteBlackList([{"site_name": site, "valid": False}])
if self.topLevelSubscription is None:
self.topLevelSubscription = subscription
logging.info("Top level subscription %s created for %s", subscription["id"], self.wmSpec.name())
else:
logging.info("Child subscription %s created for %s", subscription["id"], self.wmSpec.name())
outputModules = task.getOutputModulesForTask()
ignoredOutputModules = task.getIgnoredOutputModulesForTask()
for outputModule in outputModules:
for outputModuleName in outputModule.listSections_():
if outputModuleName in ignoredOutputModules:
msg = "%s has %s as IgnoredOutputModule, skipping fileset creation."
logging.info(msg, task.getPathName(), outputModuleName)
continue
dataTier = getattr(getattr(outputModule, outputModuleName), "dataTier", '')
filesetName = self.outputFilesetName(task, outputModuleName, dataTier)
outputFileset = Fileset(filesetName)
outputFileset.create()
outputFileset.markOpen(True)
mergedOutputFileset = None
for childTask in task.childTaskIterator():
if childTask.data.input.outputModule == outputModuleName:
childDatatier = getattr(childTask.data.input, 'dataTier', '')
if childTask.taskType() in ["Cleanup", "Merge"] and childDatatier != dataTier:
continue
elif childTask.taskType() == "Merge" and childDatatier == dataTier:
filesetName = self.outputFilesetName(childTask, "Merged", dataTier)
mergedOutputFileset = Fileset(filesetName)
mergedOutputFileset.create()
mergedOutputFileset.markOpen(True)
primaryDataset = getattr(getattr(outputModule, outputModuleName), "primaryDataset", None)
if primaryDataset is not None:
self.mergeOutputMapping[mergedOutputFileset.id] = primaryDataset
self._createSubscriptionsInWMBS(childTask, outputFileset, alternativeFilesetClose)
if mergedOutputFileset is None:
workflow.addOutput(outputModuleName + dataTier, outputFileset,
outputFileset)
else:
workflow.addOutput(outputModuleName + dataTier, outputFileset,
mergedOutputFileset)
return
