def _splitWork(self,
wmspec,
parentQueueId=None,
data=None,
mask=None,
team=None):
"""
Split work from a parent into WorkQeueueElements.
If data param supplied use that rather than getting input data from
wmspec. Used for instance when global splits by Block (avoids having to
modify wmspec block whitelist - thus all appear as same wf in wmbs)
mask can be used to specify i.e. event range.
"""
totalUnits = []
totalToplevelJobs = 0
totalEvents = 0
totalLumis = 0
totalFiles = 0
# split each top level task into constituent work elements
# get the acdc server and db name
for topLevelTask in wmspec.taskIterator():
spec = getWorkloadFromTask(topLevelTask)
policyName = spec.startPolicy()
if not policyName:
raise RuntimeError(
"WMSpec doesn't define policyName, current value: '%s'" %
policyName)
policy = startPolicy(policyName, self.params['SplittingMapping'])
self.logger.info('Splitting %s with policy %s params = %s' %
(topLevelTask.getPathName(), policyName,
self.params['SplittingMapping']))
units = policy(spec, topLevelTask, data, mask)
for unit in units:
msg = 'Queuing element %s for %s with %d job(s) split with %s' % (
unit.id, unit['Task'].getPathName(), unit['Jobs'],
policyName)
if unit['Inputs']:
msg += ' on %s' % unit['Inputs'].keys()[0]
if unit['Mask']:
msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'],
unit['Mask']['LastEvent'])
self.logger.info(msg)
totalToplevelJobs += unit['Jobs']
totalEvents += unit['NumberOfEvents']
totalLumis += unit['NumberOfLumis']
totalFiles += unit['NumberOfFiles']
totalUnits.extend(units)
return totalUnits, {
'total_jobs': totalToplevelJobs,
'input_events': totalEvents,
'input_lumis': totalLumis,
'input_num_files': totalFiles
}
def _splitWork(self, wmspec, parentQueueId = None,
data = None, mask = None, team = None):
"""
Split work from a parent into WorkQeueueElements.
If data param supplied use that rather than getting input data from
wmspec. Used for instance when global splits by Block (avoids having to
modify wmspec block whitelist - thus all appear as same wf in wmbs)
mask can be used to specify i.e. event range.
"""
totalUnits = []
totalToplevelJobs = 0
totalEvents = 0
totalLumis = 0
totalFiles = 0
# split each top level task into constituent work elements
# get the acdc server and db name
for topLevelTask in wmspec.taskIterator():
spec = getWorkloadFromTask(topLevelTask)
policyName = spec.startPolicy()
if not policyName:
raise RuntimeError("WMSpec doesn't define policyName, current value: '%s'" % policyName)
policy = startPolicy(policyName, self.params['SplittingMapping'])
self.logger.info('Splitting %s with policy %s params = %s' % (topLevelTask.getPathName(),
policyName, self.params['SplittingMapping']))
units = policy(spec, topLevelTask, data, mask)
for unit in units:
msg = 'Queuing element %s for %s with %d job(s) split with %s' % (unit.id,
unit['Task'].getPathName(), unit['Jobs'], policyName)
if unit['Inputs']:
msg += ' on %s' % unit['Inputs'].keys()[0]
if unit['Mask']:
msg += ' on events %d-%d' % (unit['Mask']['FirstEvent'], unit['Mask']['LastEvent'])
self.logger.info(msg)
totalToplevelJobs += unit['Jobs']
totalEvents += unit['NumberOfEvents']
totalLumis += unit['NumberOfLumis']
totalFiles += unit['NumberOfFiles']
totalUnits.extend(units)
return totalUnits, {'total_jobs': totalToplevelJobs, 'input_events': totalEvents, 'input_lumis': totalLumis, 'input_num_files': totalFiles}