Exemplo n.º 1
0
    def handle (self, *args, **options):
        opts = settings.init ()

        os.chdir (opts['WorkDir'])
        tasks.init (opts['TaskClassesFile'])

        total = updated = 0

        for t in Task.objects.all ():
            groupName = tasks.classify (t.name)
            total += 1
            if groupName != t.taskGroup.name:
                group = TaskGroup.objects.get_or_create (name=groupName)[0]
                t.taskGroup = group
                t.save ()
                updated += 1
        print "Processed %d tasks, updated %d" % (total, updated)
Exemplo n.º 2
0
import os
import sys
import logging
import time

from counters.lib import parser
from counters.lib import data
from counters.lib import watcher
from counters.lib import tasks
from counters.lib import settings

opts = settings.init ()

os.chdir (opts['WorkDir'])

tasks.init (opts['TaskClassesFile'])

JT_History = opts['JobTrackerHistoryDir']
watch_interval = int (opts['WatcherInterval'])

logging.basicConfig (format="%(asctime)s: %(message)s", level=logging.INFO)

w = watcher.HadoopWatcher (JT_History, JT_History + "/done", opts['WatcherStateFile'])
importer = data.CounterDataImporter ()

iteration = 0

while True:
    logging.info ("Process watching directories")
    fresh = w.process ()
    if len (fresh) > 0: