def gen(): while not self.exit_flag.is_set(): client_id, msg = worker.recv_multipart() msg = jsonapi.loads(msg) self.logger.info('new job %s, size: %d' % (client_id, len(msg))) if BertClient.is_valid_input(msg): tmp_f = list(convert_lst_to_features(msg, self.max_seq_len, self.tokenizer)) yield { 'client_id': client_id, 'input_ids': [f.input_ids for f in tmp_f], 'input_mask': [f.input_mask for f in tmp_f], 'input_type_ids': [f.input_type_ids for f in tmp_f] } else: self.logger.error('unsupported type of job %s! sending back None' % client_id)
def gen(): while not self.exit_flag.is_set(): self.dest, empty, msg = worker.recv_multipart() self._start_t = time.perf_counter() msg = pickle.loads(msg) if BertClient.is_valid_input(msg): tmp_f = list( convert_lst_to_features(msg, self.max_seq_len, self.tokenizer)) yield { 'input_ids': [f.input_ids for f in tmp_f], 'input_mask': [f.input_mask for f in tmp_f], 'input_type_ids': [f.input_type_ids for f in tmp_f] } else: logger.warning( 'worker %s: received unsupported type! sending back None' % self.dest) worker.send_multipart([self.dest, b'', b'']) worker.close()
def gen(): while not self.exit_flag.is_set(): client_id, empty, msg = worker.recv_multipart() msg = pickle.loads(msg) self.logger.info('received %4d from %s' % (len(msg), client_id)) if BertClient.is_valid_input(msg): tmp_f = list( convert_lst_to_features(msg, self.max_seq_len, self.tokenizer)) yield { 'client_id': client_id, 'input_ids': [f.input_ids for f in tmp_f], 'input_mask': [f.input_mask for f in tmp_f], 'input_type_ids': [f.input_type_ids for f in tmp_f] } else: self.logger.warning( 'received unsupported type from %s! sending back None' % client_id) worker.send_multipart([client_id, b'', b'']) worker.close()