def performTask(self, task):
self.performing_task = task
# Creating new store
store = Store(task.index_fields)
# Reading data
self.state = 'reading data from log %s' % task.log
data = self.filekeeper.read(task.log)
logging.debug("[worker %s] read %d bytes from %s" % (task.collector_name, len(data), task.log))
# Parsing and pushing data
self.state = 'parsing data from log %s' % task.log
total = 0
parsed = 0
t1 = time.time()
for line in data.split('\n'):
record = task.parser.parseLine(line+'\n')
if task.parser.successful:
parsed += 1
# after_parse_callbacks
for callback in task.after_parse_callbacks:
func = callback[0]
args = [record] + callback[1:]
try:
func(*args)
except Exception, e:
logging.debug('[%s.%s] %s' % (func.__module__, func.__name__, repr(e)))
store.push(record)
total += 1
