def init_worker(counter: Synchronized) -> None:
"""
This function runs only under parallel mode. It initializes the
individual processes which are also called workers.
"""
global _worker_id
with counter.get_lock():
counter.value += 1
_worker_id = counter.value
"""
You can now use _worker_id.
"""
# Clear the cache
from zerver.lib.cache import get_cache_backend
cache = get_cache_backend(None)
cache.clear()
# Close all connections
connections.close_all()
destroy_test_databases(_worker_id)
create_test_databases(_worker_id)
initialize_worker_path(_worker_id)
# We manually update the upload directory path in the url regex.
from zproject.dev_urls import avatars_url
new_root = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars")
avatars_url.default_args['document_root'] = new_root
def init_worker(counter: Synchronized) -> None:
"""
This function runs only under parallel mode. It initializes the
individual processes which are also called workers.
"""
global _worker_id
with counter.get_lock():
counter.value += 1
_worker_id = counter.value
"""
You can now use _worker_id.
"""
test_classes.API_KEYS = {}
# Clear the cache
from zerver.lib.cache import get_cache_backend
cache = get_cache_backend(None)
cache.clear()
# Close all connections
connections.close_all()
destroy_test_databases(_worker_id)
create_test_databases(_worker_id)
# Every process should upload to a separate directory so that
# race conditions can be avoided.
settings.LOCAL_UPLOADS_DIR = '{}_{}'.format(settings.LOCAL_UPLOADS_DIR,
_worker_id)
def is_upload_avatar_url(url: RegexURLPattern) -> bool:
if url.regex.pattern == r'^user_avatars/(?P<path>.*)$':
return True
return False
# We manually update the upload directory path in the url regex.
from zproject import dev_urls
found = False
for url in dev_urls.urls:
if is_upload_avatar_url(url):
found = True
new_root = os.path.join(settings.LOCAL_UPLOADS_DIR, "avatars")
url.default_args['document_root'] = new_root
if not found:
print("*** Upload directory not found.")
def rate_server(host: str, port: int, client_count: Synchronized) -> None:
"""rate server"""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as socket_server:
socket_server.bind((host, port))
socket_server.listen()
while True:
conn, _ = socket_server.accept()
client_con_thread = ClientConnectionThread(conn, client_count)
client_con_thread.start()
with client_count.get_lock():
client_count.value += 1
def rate_server(host: str, port: int, client_count: Synchronized) -> None:
"""rate server"""
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as socket_server:
socket_server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
socket_server.bind((host, port))
socket_server.listen(100)
while True:
conn, addr = socket_server.accept()
client_con_thread = ClientConnectionThread(conn, addr,
client_count)
client_con_thread.start()
with client_count.get_lock():
client_count.value += 1
log_client_event(client_con_thread.ident, addr[0], addr[1],
"connect")
def server_app(shared_data: Synchronized) -> None:
while True:
time.sleep(5)
with shared_data.get_lock():
shared_data.value = shared_data.value + 10
def pipeline_in_one_thread(
tasks_queue: mp.Queue,
tasks_queue_index: int,
success_shared: mp_shared.Synchronized,
device_name: str,
db_writer_queue: mp.Queue,
task_name: str,
):
with tf.device(device_name):
# =======================================================
# add initialization part of the pipeline here
pre_processor = preprocessing.TextPreprocessor('Just for init')
if os.path.exists(os.path.join('rsc/models')):
model_dir = os.path.join('rsc/models')
MER_model = MER.MatRecognition(
model_path=os.path.join(model_dir, 'matRecognition'),
bert_path=os.path.join(model_dir, 'MATBert_config'),
mat_identify_model_path=os.path.join(model_dir,
'matIdentification'),
mat_identify_bert_path=os.path.join(model_dir, 'Bert_config'),
)
else:
MER_model = MER.MatRecognition()
# =======================================================
# success = True
success_tasks = []
error_tasks = []
with timebudget('{} queue {} by {} @ {}'.format(
task_name, tasks_queue_index, device_name,
socket.gethostname())):
while True:
records = tasks_queue.get()
if records is None:
break
if not success_shared.value:
continue
para_batch = []
meta_ids = []
for para in records:
doc = pre_processor._process(para['text'])
para_batch.append(doc.user_data['text'])
meta_ids.append(para['_id']['$oid'])
try:
results_batch = MER_model.mat_recognize(para_batch)
db_writer_queue.put((meta_ids, results_batch))
# if len(success_tasks) > 0:
# raise mp.ProcessError(f'Queue {tasks_queue_index}: Error for debug!')
except Exception as e:
results_batch = []
with success_shared.get_lock():
success_shared.value = 0
print(f'{task_name} queue {tasks_queue_index}: {e}')
raise e
if success_shared.value:
success_tasks.extend(meta_ids)
else:
error_tasks.extend(meta_ids)
