示例#1
0
def run(tasks, application):
    for task in tasks.values():
        task.start()

    application.start()

    for task in tasks.values():
        task.sync()

    t0 = time()

    application.wait()

    for task in tasks.values():
        task.join()

    return t0
示例#2
0
    def handleCounters (self, taskInstance, counterParser):
        """
        Updates intermediate counter values for a job, then merge final jobInfo counters
        """
        jobInfo, tasks = counterParser.jobInfo, counterParser.tasks

        # delete existing counters
        CounterValue.objects.filter (taskInstance=taskInstance).delete ()

        # aggregate counters
        result = {}		# tag -> value mapping
        blacklist = set ()
        whitelist = set ()

        def processCounter (result, blacklist, whitelist, counter):
            """
            Return True if counter triplet (group, tag, value) must be
            included in result list
            """
            gname, tag, val = counter
            if tag in blacklist:
                return

            white = tag in whitelist
            if white:
                result[tag] = result.get (tag, 0) + long (val)
            else:
                group = counters.classify (tag)
                if group != None:
                    result[tag] = result.get (tag, 0) + long (val)
                    whitelist.add (tag)
                else:
                    blacklist.add (tag)

        # all counter values are summed up
        for t in tasks.values ():
            t.processCounters (lambda c: processCounter (result, blacklist, whitelist, c))

        # then, we process final counters form jobInfo, but only not present in a list
        if jobInfo.counters != None:
            for gname, tag, val in jobInfo.counters:
                if not tag in result:
                    group = counters.classify (tag)
                    if group != None:
                        result[tag] = long (val)

        counterValueList = []
        for tag, val in result.iteritems ():
            if val == 0:
                continue
            counterGroup = CounterGroup.objects.get_or_create (name=counters.classify (tag))[0]
            counter = Counter.objects.get_or_create (tag=tag, name=counters.name (tag),
                                                     counterGroup=counterGroup)[0]
            counterValueList.append (CounterValue (taskInstance=taskInstance, counter=counter, value=val))
        if len (counterValueList) > 0:
            CounterValue.objects.bulk_create (counterValueList)