def consumer(work_queue, work_func):
"""
@type work_queue: multiprocessing.JoinableQueue
"""
current_process = multiprocessing.current_process()
logger.info("work thread={} started, ready to work".format(current_process))
current_calc_index = 0
processed_count = 0
continue_wrok = True
while continue_wrok:
try:
calc_index, args = work_queue.get()
if calc_index != current_calc_index:
current_calc_index = calc_index
processed_count = 0
processed_count += 1
logger.info("work thread {} processing {}th work".format(current_process, processed_count))
work_func(*args)
except BrokenPipeError as error:
# 这个一般是程序退出的时候发生的，这种情况直接退出
logger.warning("work thread={} BrokenPipeError quit job".format(current_process))
continue_wrok = False
except Exception as error:
logger.error("work thread {} error={} processed count={}".format(current_process, error, processed_count))
finally:
work_queue.task_done()
logger.info("work thread ={} stopped, processed_count={}".format(current_process, processed_count))
def producer(*args):
producer_data.work_queue.put((producer_data.calc_index, args))
producer_data.produced_count += 1
logger.info("producer put %3dth work into work queue", producer_data.produced_count)
