class Producer(threading.Thread):
def __init__(self, queue, action, name, max_num, sleep_time,
work_sleep_time, work_try_num):
super(self.__class__, self).__init__()
self.queue = queue
self.action = action
self.name = name
self.max_num = max_num
self.sleep_time = sleep_time
self.work_sleep_time = work_sleep_time
self.work_try_num = work_try_num
self.rl = LogUtil().get_logger("producer", "producer" + self.name)
if not isinstance(action, base_producer_action.ProducerAction):
raise Exception("action not extends producer base")
def run(self):
action_list = []
while True:
try:
start_time = time.clock()
if len(action_list) == 0:
action_list = self.action.queue_items()
total_items = len(action_list)
self.rl.info('get queue %s total items is %s ' %
(self.name, total_items))
while True:
if len(action_list) == 0:
break
unfinished_tasks = self.queue.unfinished_tasks
if unfinished_tasks <= self.max_num:
action = action_list.pop()
self.queue.put(action)
end_time = time.clock()
work_time = int(round(end_time - start_time))
work_mins = work_time / 60
self.rl.info('put queue %s total items is %s ,total time is %s \'s,(at %s items per mins' % \
(self.name, total_items, work_time,
int(total_items) if work_mins == 0 else round(float(total_items / work_mins), 2)))
time.sleep(self.sleep_time)
except:
self.rl.exception()
def start_work(self):
for i in range(0, self.max_num):
qc = queue_consumer.Consumer(self.queue, self.name + "_" + str(i),
self.work_sleep_time,
self.work_try_num)
qc.start()
time.sleep(5)
self.start()
class Consumer(threading.Thread):
_WORK_TRY_NUM = 0
def __init__(self, queue, name, sleep_time, work_try_num):
super(self.__class__, self).__init__()
self.queue = queue
self.name = name
self.sleep_time = sleep_time
self.work_try_num = work_try_num
Consumer._WORK_TRY_NUM = work_try_num
self.rl = LogUtil().get_logger(
'consumer', 'consumer' + self.name[:self.name.find("_")])
def run(self):
while True:
try:
# 这是一个阻塞方法
action = self.queue.get()
if not isinstance(action, base_consumer_action.ConsumerAction):
raise Exception("action not extends consumer base")
sleep_time = random.randint(0, self.sleep_time * 10) * 0.1
time.sleep(sleep_time)
action.consumer_thread_name = self.name
start_time = time.clock()
re = action.action()
end_time = time.clock()
work_time = int(round(end_time - start_time))
self.rl.info(("queue name %s finish,sleep time %s \'s,action time %s \'s"
"action retry %s times,result:%s") % \
(self.name, sleep_time, work_time, action.try_num, re.__str__() if re is not None else ""))
if not re[0] and action.try_num < self.work_try_num:
action.try_num += 1
self.queue.put(action)
self.queue.task_done()
except:
self.rl.exception()