def handle (self, *args, **options): opts = settings.init () os.chdir (opts['WorkDir']) tasks.init (opts['TaskClassesFile']) total = updated = 0 for t in Task.objects.all (): groupName = tasks.classify (t.name) total += 1 if groupName != t.taskGroup.name: group = TaskGroup.objects.get_or_create (name=groupName)[0] t.taskGroup = group t.save () updated += 1 print "Processed %d tasks, updated %d" % (total, updated)
import os import sys import logging import time from counters.lib import parser from counters.lib import data from counters.lib import watcher from counters.lib import tasks from counters.lib import settings opts = settings.init () os.chdir (opts['WorkDir']) tasks.init (opts['TaskClassesFile']) JT_History = opts['JobTrackerHistoryDir'] watch_interval = int (opts['WatcherInterval']) logging.basicConfig (format="%(asctime)s: %(message)s", level=logging.INFO) w = watcher.HadoopWatcher (JT_History, JT_History + "/done", opts['WatcherStateFile']) importer = data.CounterDataImporter () iteration = 0 while True: logging.info ("Process watching directories") fresh = w.process () if len (fresh) > 0: