def consumer(work_queue, work_func): """ @type work_queue: multiprocessing.JoinableQueue """ current_process = multiprocessing.current_process() logger.info("work thread={} started, ready to work".format(current_process)) current_calc_index = 0 processed_count = 0 continue_wrok = True while continue_wrok: try: calc_index, args = work_queue.get() if calc_index != current_calc_index: current_calc_index = calc_index processed_count = 0 processed_count += 1 logger.info("work thread {} processing {}th work".format(current_process, processed_count)) work_func(*args) except BrokenPipeError as error: # 这个一般是程序退出的时候发生的,这种情况直接退出 logger.warning("work thread={} BrokenPipeError quit job".format(current_process)) continue_wrok = False except Exception as error: logger.error("work thread {} error={} processed count={}".format(current_process, error, processed_count)) finally: work_queue.task_done() logger.info("work thread ={} stopped, processed_count={}".format(current_process, processed_count))
def producer(*args): producer_data.work_queue.put((producer_data.calc_index, args)) producer_data.produced_count += 1 logger.info("producer put %3dth work into work queue", producer_data.produced_count)