def gen():
while not self.exit_flag.is_set():
client_id, msg = worker.recv_multipart()
msg = jsonapi.loads(msg)
self.logger.info('new job %s, size: %d' % (client_id, len(msg)))
if BertClient.is_valid_input(msg):
tmp_f = list(convert_lst_to_features(msg, self.max_seq_len, self.tokenizer))
yield {
'client_id': client_id,
'input_ids': [f.input_ids for f in tmp_f],
'input_mask': [f.input_mask for f in tmp_f],
'input_type_ids': [f.input_type_ids for f in tmp_f]
}
else:
self.logger.error('unsupported type of job %s! sending back None' % client_id)
def gen():
while not self.exit_flag.is_set():
self.dest, empty, msg = worker.recv_multipart()
self._start_t = time.perf_counter()
msg = pickle.loads(msg)
if BertClient.is_valid_input(msg):
tmp_f = list(
convert_lst_to_features(msg, self.max_seq_len,
self.tokenizer))
yield {
'input_ids': [f.input_ids for f in tmp_f],
'input_mask': [f.input_mask for f in tmp_f],
'input_type_ids': [f.input_type_ids for f in tmp_f]
}
else:
logger.warning(
'worker %s: received unsupported type! sending back None'
% self.dest)
worker.send_multipart([self.dest, b'', b''])
worker.close()
def gen():
while not self.exit_flag.is_set():
client_id, empty, msg = worker.recv_multipart()
msg = pickle.loads(msg)
self.logger.info('received %4d from %s' %
(len(msg), client_id))
if BertClient.is_valid_input(msg):
tmp_f = list(
convert_lst_to_features(msg, self.max_seq_len,
self.tokenizer))
yield {
'client_id': client_id,
'input_ids': [f.input_ids for f in tmp_f],
'input_mask': [f.input_mask for f in tmp_f],
'input_type_ids': [f.input_type_ids for f in tmp_f]
}
else:
self.logger.warning(
'received unsupported type from %s! sending back None'
% client_id)
worker.send_multipart([client_id, b'', b''])
worker.close()
