def run(tasks, application):
for task in tasks.values():
task.start()
application.start()
for task in tasks.values():
task.sync()
t0 = time()
application.wait()
for task in tasks.values():
task.join()
return t0
def handleCounters (self, taskInstance, counterParser):
"""
Updates intermediate counter values for a job, then merge final jobInfo counters
"""
jobInfo, tasks = counterParser.jobInfo, counterParser.tasks
# delete existing counters
CounterValue.objects.filter (taskInstance=taskInstance).delete ()
# aggregate counters
result = {} # tag -> value mapping
blacklist = set ()
whitelist = set ()
def processCounter (result, blacklist, whitelist, counter):
"""
Return True if counter triplet (group, tag, value) must be
included in result list
"""
gname, tag, val = counter
if tag in blacklist:
return
white = tag in whitelist
if white:
result[tag] = result.get (tag, 0) + long (val)
else:
group = counters.classify (tag)
if group != None:
result[tag] = result.get (tag, 0) + long (val)
whitelist.add (tag)
else:
blacklist.add (tag)
# all counter values are summed up
for t in tasks.values ():
t.processCounters (lambda c: processCounter (result, blacklist, whitelist, c))
# then, we process final counters form jobInfo, but only not present in a list
if jobInfo.counters != None:
for gname, tag, val in jobInfo.counters:
if not tag in result:
group = counters.classify (tag)
if group != None:
result[tag] = long (val)
counterValueList = []
for tag, val in result.iteritems ():
if val == 0:
continue
counterGroup = CounterGroup.objects.get_or_create (name=counters.classify (tag))[0]
counter = Counter.objects.get_or_create (tag=tag, name=counters.name (tag),
counterGroup=counterGroup)[0]
counterValueList.append (CounterValue (taskInstance=taskInstance, counter=counter, value=val))
if len (counterValueList) > 0:
CounterValue.objects.bulk_create (counterValueList)