def server_app(shared_data: Synchronized) -> None:
while True:
time.sleep(5)
with shared_data.get_lock():
shared_data.value = shared_data.value + 10
def pipeline_in_one_thread(
tasks_queue: mp.Queue,
tasks_queue_index: int,
success_shared: mp_shared.Synchronized,
device_name: str,
db_writer_queue: mp.Queue,
task_name: str,
):
with tf.device(device_name):
# =======================================================
# add initialization part of the pipeline here
pre_processor = preprocessing.TextPreprocessor('Just for init')
if os.path.exists(os.path.join('rsc/models')):
model_dir = os.path.join('rsc/models')
MER_model = MER.MatRecognition(
model_path=os.path.join(model_dir, 'matRecognition'),
bert_path=os.path.join(model_dir, 'MATBert_config'),
mat_identify_model_path=os.path.join(model_dir,
'matIdentification'),
mat_identify_bert_path=os.path.join(model_dir, 'Bert_config'),
)
else:
MER_model = MER.MatRecognition()
# =======================================================
# success = True
success_tasks = []
error_tasks = []
with timebudget('{} queue {} by {} @ {}'.format(
task_name, tasks_queue_index, device_name,
socket.gethostname())):
while True:
records = tasks_queue.get()
if records is None:
break
if not success_shared.value:
continue
para_batch = []
meta_ids = []
for para in records:
doc = pre_processor._process(para['text'])
para_batch.append(doc.user_data['text'])
meta_ids.append(para['_id']['$oid'])
try:
results_batch = MER_model.mat_recognize(para_batch)
db_writer_queue.put((meta_ids, results_batch))
# if len(success_tasks) > 0:
# raise mp.ProcessError(f'Queue {tasks_queue_index}: Error for debug!')
except Exception as e:
results_batch = []
with success_shared.get_lock():
success_shared.value = 0
print(f'{task_name} queue {tasks_queue_index}: {e}')
raise e
if success_shared.value:
success_tasks.extend(meta_ids)
else:
error_tasks.extend(meta_ids)
