def handle (self, *args, **options):
opts = settings.init ()
os.chdir (opts['WorkDir'])
tasks.init (opts['TaskClassesFile'])
total = updated = 0
for t in Task.objects.all ():
groupName = tasks.classify (t.name)
total += 1
if groupName != t.taskGroup.name:
group = TaskGroup.objects.get_or_create (name=groupName)[0]
t.taskGroup = group
t.save ()
updated += 1
print "Processed %d tasks, updated %d" % (total, updated)
import os
import sys
import logging
import time
from counters.lib import parser
from counters.lib import data
from counters.lib import watcher
from counters.lib import tasks
from counters.lib import settings
opts = settings.init ()
os.chdir (opts['WorkDir'])
tasks.init (opts['TaskClassesFile'])
JT_History = opts['JobTrackerHistoryDir']
watch_interval = int (opts['WatcherInterval'])
logging.basicConfig (format="%(asctime)s: %(message)s", level=logging.INFO)
w = watcher.HadoopWatcher (JT_History, JT_History + "/done", opts['WatcherStateFile'])
importer = data.CounterDataImporter ()
iteration = 0
while True:
logging.info ("Process watching directories")
fresh = w.process ()
if len (fresh) > 0:
