def __init__(self, task, job):
"""
Init some stuff
"""
#Basic task/job objects
self.task = task
self.workload = getWorkloadFromTask(task)
self.job = job
#Dashboard server interface info
self.publisher = None
self.destinations = {}
self.server = None
#Job ids
self.taskName = 'wmagent_%s' % self.workload.name()
self.jobName = '%s_%i' % (job['name'], job['retry_count'])
#Step counter
self.stepCount = 0
#Job ending report stuff
self.jobSuccess = 0
self.jobStarted = False
self.failedStep = None
self.lastStep = None
self.WrapperWCTime = 0
self.WrapperCPUTime = 0
#Utility
self.tsFormat = '%Y-%m-%d %H:%M:%S'
return
def __call__(self, job, task, report, export = True, startTime = 'None', endTime = 'None'):
"""
__call__
Does everything. Basically it runs all the other functions.
Accepts three arguments, a DS Job object, a WMTaskHelper,
and a FwkJobReport.Report
"""
self.document = minidom.Document() # This is the end product, the master report
self.job = job
self.task = task
self.report = report
self.workload = getWorkloadFromTask(task)
self.startTime = startTime
self.endTime = endTime
self.url = getattr(task.data, 'dashboardURL',
"http://dashb-cmspa.cern.ch/dashboard/request.py/getPAinfo")
self.createDocument()
if export:
self.exportDocument()
return
def __init__(self, task, job):
"""
Init some stuff
"""
#Basic task/job objects
self.task = task
self.workload = getWorkloadFromTask(task)
self.job = job
#Dashboard server interface info
self.publisher = None
self.destinations = {}
self.server = None
#Job ids
self.taskName = 'wmagent_%s' % self.workload.name()
self.jobName = '%s_%i' % (job['name'], job['retry_count'])
#Step counter
self.stepCount = 0
#Job ending report stuff
self.jobSuccess = 0
self.jobStarted = False
self.failedStep = None
self.lastStep = None
self.WrapperWCTime = 0
self.WrapperCPUTime = 0
#Utility
self.tsFormat = '%Y-%m-%d %H:%M:%S'
return
def __call__(self, job, task, report, export = True, startTime = 'None', endTime = 'None'):
"""
__call__
Does everything. Basically it runs all the other functions.
Accepts three arguments, a DS Job object, a WMTaskHelper,
and a FwkJobReport.Report
"""
self.document = minidom.Document() # This is the end product, the master report
self.job = job
self.task = task
self.report = report
self.workload = getWorkloadFromTask(task)
self.startTime = startTime
self.endTime = endTime
self.url = getattr(task.data, 'dashboardURL',
"http://dashb-cmspa.cern.ch/dashboard/request.py/getPAinfo")
self.createDocument()
if export:
self.exportDocument()
return
def testTwoTaskTree(self):
workload = TestSpecs.twoTaskTree()
tasks = workload.listAllTaskNames()
parenttask = workload.getTask("FirstTask")
findWorkload = getWorkloadFromTask(parenttask)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) )
self.assertTrue( isinstance( parenttask, WMCore.WMSpec.WMTask.WMTaskHelper ) )
task = workload.getTask("SecondTask")
#print task.data
self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) )
steps = task.listAllStepNames()
# there should be a way to do this with iteration that would be neater
step = task.getStep(steps[0])
self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) )
step2= task.getStep(steps[1])
self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) )
step3= task.getStep(steps[2])
self.assertTrue( isinstance( step3, WMCore.WMSpec.WMStep.WMStepHelper ) )
step4= task.getStep(steps[3])
self.assertTrue( isinstance( step4, WMCore.WMSpec.WMStep.WMStepHelper ) )
self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
def testTwoTaskTree(self):
workload = TestSpecs.twoTaskTree()
tasks = workload.listAllTaskNames()
parenttask = workload.getTask("FirstTask")
findWorkload = getWorkloadFromTask(parenttask)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue(
isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper))
self.assertTrue(
isinstance(parenttask, WMCore.WMSpec.WMTask.WMTaskHelper))
task = workload.getTask("SecondTask")
#print task.data
self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper))
steps = task.listAllStepNames()
# there should be a way to do this with iteration that would be neater
step = task.getStep(steps[0])
self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper))
step2 = task.getStep(steps[1])
self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper))
step3 = task.getStep(steps[2])
self.assertTrue(isinstance(step3, WMCore.WMSpec.WMStep.WMStepHelper))
step4 = task.getStep(steps[3])
self.assertTrue(isinstance(step4, WMCore.WMSpec.WMStep.WMStepHelper))
self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
def _splitWork(self,
wmspec,
parentQueueId=None,
data=None,
mask=None,
team=None):
"""
Split work from a parent into WorkQeueueElements.
If data param supplied use that rather than getting input data from
wmspec. Used for instance when global splits by Block (avoids having to
modify wmspec block whitelist - thus all appear as same wf in wmbs)
mask can be used to specify i.e. event range.
"""
totalUnits = []
totalToplevelJobs = 0
totalEvents = 0
totalLumis = 0
totalFiles = 0
# split each top level task into constituent work elements
# get the acdc server and db name
for topLevelTask in wmspec.taskIterator():
spec = getWorkloadFromTask(topLevelTask)
policyName = spec.startPolicy()
if not policyName:
raise RuntimeError(
"WMSpec doesn't define policyName, current value: '%s'" %
policyName)
policy = startPolicy(policyName, self.params['SplittingMapping'])
self.logger.info('Splitting %s with policy %s params = %s' %
(topLevelTask.getPathName(), policyName,
self.params['SplittingMapping']))
units = policy(spec, topLevelTask, data, mask)
for unit in units:
msg = 'Queuing element %s for %s with %d job(s) split with %s' % (
unit.id, unit['Task'].getPathName(), unit['Jobs'],
policyName)
if unit['Inputs']:
msg += ' on %s' % unit['Inputs'].keys()[0]
if unit['Mask']:
msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'],
unit['Mask']['LastEvent'])
self.logger.info(msg)
totalToplevelJobs += unit['Jobs']
totalEvents += unit['NumberOfEvents']
totalLumis += unit['NumberOfLumis']
totalFiles += unit['NumberOfFiles']
totalUnits.extend(units)
return totalUnits, {
'total_jobs': totalToplevelJobs,
'input_events': totalEvents,
'input_lumis': totalLumis,
'input_num_files': totalFiles
}
def testOneTaskTwoSTep(self):
workload = TestSpecs.oneTaskTwoStep()
tasks = workload.listAllTaskNames()
task = workload.getTask(tasks[0])
findWorkload = getWorkloadFromTask(task)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue( isinstance( workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper ) )
self.assertTrue( isinstance( task, WMCore.WMSpec.WMTask.WMTaskHelper ) )
steps = task.listAllStepNames()
step = task.getStep(steps[0])
self.assertTrue( isinstance( step, WMCore.WMSpec.WMStep.WMStepHelper ) )
step2= task.getStep(steps[1])
self.assertTrue( isinstance( step2, WMCore.WMSpec.WMStep.WMStepHelper ) )
self.assertEqual( id(getTaskFromStep(step)), id(getTaskFromStep(step2)) )
def testOneTaskTwoSTep(self):
workload = TestSpecs.oneTaskTwoStep()
tasks = workload.listAllTaskNames()
task = workload.getTask(tasks[0])
findWorkload = getWorkloadFromTask(task)
self.assertEqual(id(findWorkload.data), id(workload.data))
self.assertTrue(
isinstance(workload, WMCore.WMSpec.WMWorkload.WMWorkloadHelper))
self.assertTrue(isinstance(task, WMCore.WMSpec.WMTask.WMTaskHelper))
steps = task.listAllStepNames()
step = task.getStep(steps[0])
self.assertTrue(isinstance(step, WMCore.WMSpec.WMStep.WMStepHelper))
step2 = task.getStep(steps[1])
self.assertTrue(isinstance(step2, WMCore.WMSpec.WMStep.WMStepHelper))
self.assertEqual(id(getTaskFromStep(step)), id(getTaskFromStep(step2)))
def __init__(self, task, job):
"""
Init some stuff
"""
self.task = task
self.workload = getWorkloadFromTask(task)
self.job = job
self.publisher = None
self.destinations = {}
self.server = None
dict.__init__(self)
self.setdefault("Application", None)
self.setdefault("ApplicationVersion", None)
self.setdefault("GridJobID", None)
self.setdefault("LocalBatchID", None)
self.setdefault("GridUser", None)
self.setdefault("User" , self.workload.getOwner().get('name', 'WMAgent'))
self.setdefault("JSTool","WMAgent")
self.setdefault("NbEvPerRun", 0)
self.setdefault("NodeName", None)
self.setdefault("Scheduler", None)
self.setdefault("TaskType", self.task.taskType())
self.setdefault("NSteps", 0)
self.setdefault("VO", "CMS")
self.setdefault("TargetCE", None)
self.setdefault("RBname", None)
self.setdefault("JSToolUI" , None) # Can't set here, see bug #64232
self.taskName = 'wmagent_%s' % self.workload.name()
self.jobName = '%s_%i' % (job['name'], job['retry_count'])
self.jobSuccess = 0
return
def _splitWork(self, wmspec, parentQueueId = None,
data = None, mask = None, team = None):
"""
Split work from a parent into WorkQeueueElements.
If data param supplied use that rather than getting input data from
wmspec. Used for instance when global splits by Block (avoids having to
modify wmspec block whitelist - thus all appear as same wf in wmbs)
mask can be used to specify i.e. event range.
"""
totalUnits = []
totalToplevelJobs = 0
totalEvents = 0
totalLumis = 0
totalFiles = 0
# split each top level task into constituent work elements
# get the acdc server and db name
for topLevelTask in wmspec.taskIterator():
spec = getWorkloadFromTask(topLevelTask)
policyName = spec.startPolicy()
if not policyName:
raise RuntimeError("WMSpec doesn't define policyName, current value: '%s'" % policyName)
policy = startPolicy(policyName, self.params['SplittingMapping'])
self.logger.info('Splitting %s with policy %s params = %s' % (topLevelTask.getPathName(),
policyName, self.params['SplittingMapping']))
units = policy(spec, topLevelTask, data, mask)
for unit in units:
msg = 'Queuing element %s for %s with %d job(s) split with %s' % (unit.id,
unit['Task'].getPathName(), unit['Jobs'], policyName)
if unit['Inputs']:
msg += ' on %s' % unit['Inputs'].keys()[0]
if unit['Mask']:
msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'], unit['Mask']['LastEvent'])
self.logger.info(msg)
totalToplevelJobs += unit['Jobs']
totalEvents += unit['NumberOfEvents']
totalLumis += unit['NumberOfLumis']
totalFiles += unit['NumberOfFiles']
totalUnits.extend(units)
return totalUnits, {'total_jobs': totalToplevelJobs, 'input_events': totalEvents, 'input_lumis': totalLumis, 'input_num_files': totalFiles}
