class ProcessorWorker(Process):
def __init__(self, task_queue, callback, count=None):
super(ProcessorWorker, self).__init__()
self.task_queue = task_queue
self.callback = callback
self.storage = DbHandler()
self.callback = partial(callback, self.storage)
self.count = count
log.info("process worker %s inited with count: %s" % (self.name, self.count))
def run(self):
count = 1
c_t = 0
s_t = 0
while True:
to_process = self.task_queue.get()
message = to_process.get("message")
task = to_process.get("task")
start = time()
result = self.callback(message)
cbc = time()
self.storage.save_timeline_element(result, task, deferred_save=True)
end = time()
count += 1
c_t += cbc - start
s_t += end - cbc
if count % (self.count / 10 if self.count else 10) == 0:
log.info("was process and save %s tweets\n pa: %s, sa: %s" % (count, c_t / count, s_t / count))
if self.count and count > self.count:
log.info("will end my work, i was create %s iterations \n pa: %s, sa: %s" % (
count, c_t / count, s_t / count))
break
