def bot_func(bot, update, args): text = " ".join(args) words = utils.tokenize(text) seq_1 = data.encode_words(words, emb_dict) input_seq = model.pack_input(seq_1, net.emb) enc = net.encode(input_seq) if prog_args.sample: _, tokens = net.decode_chain_sampling(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) else: _, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) if tokens[-1] == end_token: tokens = tokens[:-1] reply = data.decode_words(tokens, rev_emb_dict) if reply: reply_text = utils.untokenize(reply) bot.send_message(chat_id=update.message.chat_id, text=reply_text)
def run_test(test_data, net, end_token, device="cpu"): bleu_sum = 0.0 bleu_count = 0 for p1, p2 in test_data: input_seq = model.pack_input(p1, net.emb, device) enc = net.encode(input_seq) _, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) bleu_sum += utils.calc_bleu(tokens, p2[1:]) bleu_count += 1 return bleu_sum / bleu_count
def run_test(test_data, net, end_token, cuda=False): bleu_sum = 0.0 bleu_count = 0 for p1, p2 in test_data: input_seq = model.pack_input(p1, net.emb, cuda) enc = net.encode(input_seq) _, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) ref_indices = [indices[1:] for indices in p2] bleu_sum += utils.calc_bleu_many(tokens, ref_indices) bleu_count += 1 return bleu_sum / bleu_count
def words_to_words(words, emb_dict, rev_emb_dict, net, use_sampling=False): tokens = data.encode_words(words, emb_dict) input_seq = model.pack_input(tokens, net.emb) enc = net.encode(input_seq) end_token = emb_dict[data.END_TOKEN] if use_sampling: _, out_tokens = net.decode_chain_sampling(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) else: _, out_tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) if out_tokens[-1] == end_token: out_tokens = out_tokens[:-1] out_words = data.decode_words(out_tokens, rev_emb_dict) return out_words
def run_test(test_data, net, rev_emb_dict, end_token, device="cuda"): argmax_reward_sum = 0.0 argmax_reward_count = 0.0 # p1 is one sentence, p2 is sentence list. for p1, p2 in test_data: # Transform sentence to padded embeddings. input_seq = model.pack_input(p1, net.emb, device) # Get hidden states from encoder. enc = net.encode(input_seq) # Decode sequence by feeding predicted token to the net again. Act greedily. # Return N*outputvocab, N output token indices. _, tokens = net.decode_chain_argmax(enc, net.emb(beg_token), seq_len=data.MAX_TOKENS, stop_at_token=end_token) # Show what the output action sequence is. action_tokens = [] for temp_idx in tokens: if temp_idx in rev_emb_dict and rev_emb_dict.get( temp_idx) != '#END': action_tokens.append(str(rev_emb_dict.get(temp_idx)).upper()) argmax_reward_sum += float(utils.calc_True_Reward(action_tokens, p2)) argmax_reward_count += 1 return float(argmax_reward_sum) / float(argmax_reward_count)
def run_test(test_data, net, end_token, device="cuda"): bleu_sum = 0.0 bleu_count = 0 # p1 is one sentence, p2 is sentence list. for p1, p2 in test_data: # Transform sentence to padded embeddings. input_seq = model.pack_input(p1, net.emb, device) # Get hidden states from encoder. enc = net.encode(input_seq) # Decode sequence by feeding predicted token to the net again. Act greedily. # Return N*outputvocab, N output token indices. _, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) ref_indices = [ # Remove #BEG from sentence. indices[1:] for indices in p2 ] # BEG is not included in tokens. # Accept several reference sentences and return the one with the best score. bleu_sum += utils.calc_bleu_many(tokens, ref_indices) bleu_count += 1 return bleu_sum / bleu_count
net = net.cuda() model_path = '../data/saves/' + str(args.name) + '/' + str(args.model) net.load_state_dict((torch.load(model_path))) end_token = emb_dict[data.END_TOKEN] seq_count = 0 correct_count = 0 sum_bleu = 0.0 test_dataset_count = 0 token_string_list = list() refer_string_list = list() # seq_1是輸入,targets是references,可能有多個; for seq_1, targets in train_data: test_dataset_count += 1 input_seq = model.pack_input(seq_1, net.emb) enc = net.encode(input_seq) _, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) references = [seq[1:] for seq in targets] # references = [seq[1:] if seq[1:] != '' else ['NONE'] for seq in targets] token_string, reference_string = '', '' for token in tokens: if token in rev_emb_dict and rev_emb_dict.get(token) != '#END': token_string += str(rev_emb_dict.get(token)).upper() + ' ' token_string = token_string.strip() # log.info("%d PREDICT: %s", test_dataset_count, token_string) token_string_list.append( str(test_dataset_count) + ': ' + token_string + '\n')
parser.add_argument("--data", required=True, help="Category to use for training. Empty string to train on full dataset") parser.add_argument("-m", "--model", required=True, help="Model name to load") args = parser.parse_args() phrase_pairs, emb_dict = data.load_data(args.data) log.info("Obtained %d phrase pairs with %d uniq words", len(phrase_pairs), len(emb_dict)) train_data = data.encode_phrase_pairs(phrase_pairs, emb_dict) train_data = data.group_train_data(train_data) rev_emb_dict = {idx: word for word, idx in emb_dict.items()} net = model.PhraseModel(emb_size=model.EMBEDDING_DIM, dict_size=len(emb_dict), hid_size=model.HIDDEN_STATE_SIZE) net.load_state_dict(torch.load(args.model)) end_token = emb_dict[data.END_TOKEN] seq_count = 0 sum_bleu = 0.0 for seq_1, targets in train_data: input_seq = model.pack_input(seq_1, net.emb) enc = net.encode(input_seq) _, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS, stop_at_token=end_token) references = [seq[1:] for seq in targets] bleu = utils.calc_bleu_many(tokens, references) sum_bleu += bleu seq_count += 1 log.info("Processed %d phrases, mean BLEU = %.4f", seq_count, sum_bleu / seq_count)