def performTask(self, task): self.performing_task = task # Creating new store store = Store(task.index_fields) # Reading data self.state = 'reading data from log %s' % task.log data = self.filekeeper.read(task.log) logging.debug("[worker %s] read %d bytes from %s" % (task.collector_name, len(data), task.log)) # Parsing and pushing data self.state = 'parsing data from log %s' % task.log total = 0 parsed = 0 t1 = time.time() for line in data.split('\n'): record = task.parser.parseLine(line+'\n') if task.parser.successful: parsed += 1 # after_parse_callbacks for callback in task.after_parse_callbacks: func = callback[0] args = [record] + callback[1:] try: func(*args) except Exception, e: logging.debug('[%s.%s] %s' % (func.__module__, func.__name__, repr(e))) store.push(record) total += 1