def server_app(shared_data: Synchronized) -> None: while True: time.sleep(5) with shared_data.get_lock(): shared_data.value = shared_data.value + 10
def pipeline_in_one_thread( tasks_queue: mp.Queue, tasks_queue_index: int, success_shared: mp_shared.Synchronized, device_name: str, db_writer_queue: mp.Queue, task_name: str, ): with tf.device(device_name): # ======================================================= # add initialization part of the pipeline here pre_processor = preprocessing.TextPreprocessor('Just for init') if os.path.exists(os.path.join('rsc/models')): model_dir = os.path.join('rsc/models') MER_model = MER.MatRecognition( model_path=os.path.join(model_dir, 'matRecognition'), bert_path=os.path.join(model_dir, 'MATBert_config'), mat_identify_model_path=os.path.join(model_dir, 'matIdentification'), mat_identify_bert_path=os.path.join(model_dir, 'Bert_config'), ) else: MER_model = MER.MatRecognition() # ======================================================= # success = True success_tasks = [] error_tasks = [] with timebudget('{} queue {} by {} @ {}'.format( task_name, tasks_queue_index, device_name, socket.gethostname())): while True: records = tasks_queue.get() if records is None: break if not success_shared.value: continue para_batch = [] meta_ids = [] for para in records: doc = pre_processor._process(para['text']) para_batch.append(doc.user_data['text']) meta_ids.append(para['_id']['$oid']) try: results_batch = MER_model.mat_recognize(para_batch) db_writer_queue.put((meta_ids, results_batch)) # if len(success_tasks) > 0: # raise mp.ProcessError(f'Queue {tasks_queue_index}: Error for debug!') except Exception as e: results_batch = [] with success_shared.get_lock(): success_shared.value = 0 print(f'{task_name} queue {tasks_queue_index}: {e}') raise e if success_shared.value: success_tasks.extend(meta_ids) else: error_tasks.extend(meta_ids)