def bot_func(bot, update, args):
text = " ".join(args)
words = utils.tokenize(text)
seq_1 = data.encode_words(words, emb_dict)
input_seq = model.pack_input(seq_1, net.emb)
enc = net.encode(input_seq)
if prog_args.sample:
_, tokens = net.decode_chain_sampling(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
else:
_, tokens = net.decode_chain_argmax(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
if tokens[-1] == end_token:
tokens = tokens[:-1]
reply = data.decode_words(tokens, rev_emb_dict)
if reply:
reply_text = utils.untokenize(reply)
bot.send_message(chat_id=update.message.chat_id, text=reply_text)
def run_test(test_data, net, end_token, device="cpu"):
bleu_sum = 0.0
bleu_count = 0
for p1, p2 in test_data:
input_seq = model.pack_input(p1, net.emb, device)
enc = net.encode(input_seq)
_, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
bleu_sum += utils.calc_bleu(tokens, p2[1:])
bleu_count += 1
return bleu_sum / bleu_count
def run_test(test_data, net, end_token, cuda=False):
bleu_sum = 0.0
bleu_count = 0
for p1, p2 in test_data:
input_seq = model.pack_input(p1, net.emb, cuda)
enc = net.encode(input_seq)
_, tokens = net.decode_chain_argmax(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
ref_indices = [indices[1:] for indices in p2]
bleu_sum += utils.calc_bleu_many(tokens, ref_indices)
bleu_count += 1
return bleu_sum / bleu_count
def words_to_words(words, emb_dict, rev_emb_dict, net, use_sampling=False):
tokens = data.encode_words(words, emb_dict)
input_seq = model.pack_input(tokens, net.emb)
enc = net.encode(input_seq)
end_token = emb_dict[data.END_TOKEN]
if use_sampling:
_, out_tokens = net.decode_chain_sampling(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
else:
_, out_tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
if out_tokens[-1] == end_token:
out_tokens = out_tokens[:-1]
out_words = data.decode_words(out_tokens, rev_emb_dict)
return out_words
def bot_func(bot, update, args):
text = " ".join(args)
words = utils.tokenize(text)
seq_1 = data.encode_words(words, emb_dict)
input_seq = model.pack_input(seq_1, net.emb)
enc = net.encode(input_seq)
if prog_args.sample:
_, tokens = net.decode_chain_sampling(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
else:
_, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1], seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
if tokens[-1] == end_token:
tokens = tokens[:-1]
reply = data.decode_words(tokens, rev_emb_dict)
if reply:
reply_text = utils.untokenize(reply)
bot.send_message(chat_id=update.message.chat_id, text=reply_text)
def words_to_words(words, emb_dict, rev_emb_dict, net, use_sampling=False):
tokens = data.encode_words(words, emb_dict)
input_seq = model.pack_input(tokens, net.emb)
enc = net.encode(input_seq)
end_token = emb_dict[data.END_TOKEN]
if use_sampling:
_, out_tokens = net.decode_chain_sampling(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
else:
_, out_tokens = net.decode_chain_argmax(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
if out_tokens[-1] == end_token:
out_tokens = out_tokens[:-1]
out_words = data.decode_words(out_tokens, rev_emb_dict)
return out_words
def run_test(test_data, net, rev_emb_dict, end_token, device="cuda"):
argmax_reward_sum = 0.0
argmax_reward_count = 0.0
# p1 is one sentence, p2 is sentence list.
for p1, p2 in test_data:
# Transform sentence to padded embeddings.
input_seq = model.pack_input(p1, net.emb, device)
# Get hidden states from encoder.
enc = net.encode(input_seq)
# Decode sequence by feeding predicted token to the net again. Act greedily.
# Return N*outputvocab, N output token indices.
_, tokens = net.decode_chain_argmax(enc,
net.emb(beg_token),
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
# Show what the output action sequence is.
action_tokens = []
for temp_idx in tokens:
if temp_idx in rev_emb_dict and rev_emb_dict.get(
temp_idx) != '#END':
action_tokens.append(str(rev_emb_dict.get(temp_idx)).upper())
argmax_reward_sum += float(utils.calc_True_Reward(action_tokens, p2))
argmax_reward_count += 1
return float(argmax_reward_sum) / float(argmax_reward_count)
def run_test(test_data, net, end_token, device="cuda"):
bleu_sum = 0.0
bleu_count = 0
# p1 is one sentence, p2 is sentence list.
for p1, p2 in test_data:
# Transform sentence to padded embeddings.
input_seq = model.pack_input(p1, net.emb, device)
# Get hidden states from encoder.
enc = net.encode(input_seq)
# Decode sequence by feeding predicted token to the net again. Act greedily.
# Return N*outputvocab, N output token indices.
_, tokens = net.decode_chain_argmax(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
ref_indices = [
# Remove #BEG from sentence.
indices[1:] for indices in p2
]
# BEG is not included in tokens.
# Accept several reference sentences and return the one with the best score.
bleu_sum += utils.calc_bleu_many(tokens, ref_indices)
bleu_count += 1
return bleu_sum / bleu_count
net = net.cuda()
model_path = '../data/saves/' + str(args.name) + '/' + str(args.model)
net.load_state_dict((torch.load(model_path)))
end_token = emb_dict[data.END_TOKEN]
seq_count = 0
correct_count = 0
sum_bleu = 0.0
test_dataset_count = 0
token_string_list = list()
refer_string_list = list()
# seq_1是輸入，targets是references，可能有多個；
for seq_1, targets in train_data:
test_dataset_count += 1
input_seq = model.pack_input(seq_1, net.emb)
enc = net.encode(input_seq)
_, tokens = net.decode_chain_argmax(enc,
input_seq.data[0:1],
seq_len=data.MAX_TOKENS,
stop_at_token=end_token)
references = [seq[1:] for seq in targets]
# references = [seq[1:] if seq[1:] != '' else ['NONE'] for seq in targets]
token_string, reference_string = '', ''
for token in tokens:
if token in rev_emb_dict and rev_emb_dict.get(token) != '#END':
token_string += str(rev_emb_dict.get(token)).upper() + ' '
token_string = token_string.strip()
# log.info("%d PREDICT: %s", test_dataset_count, token_string)
token_string_list.append(
str(test_dataset_count) + ': ' + token_string + '\n')
parser.add_argument("--data", required=True,
help="Category to use for training. Empty string to train on full dataset")
parser.add_argument("-m", "--model", required=True, help="Model name to load")
args = parser.parse_args()
phrase_pairs, emb_dict = data.load_data(args.data)
log.info("Obtained %d phrase pairs with %d uniq words", len(phrase_pairs), len(emb_dict))
train_data = data.encode_phrase_pairs(phrase_pairs, emb_dict)
train_data = data.group_train_data(train_data)
rev_emb_dict = {idx: word for word, idx in emb_dict.items()}
net = model.PhraseModel(emb_size=model.EMBEDDING_DIM, dict_size=len(emb_dict), hid_size=model.HIDDEN_STATE_SIZE)
net.load_state_dict(torch.load(args.model))
end_token = emb_dict[data.END_TOKEN]
seq_count = 0
sum_bleu = 0.0
for seq_1, targets in train_data:
input_seq = model.pack_input(seq_1, net.emb)
enc = net.encode(input_seq)
_, tokens = net.decode_chain_argmax(enc, input_seq.data[0:1],
seq_len=data.MAX_TOKENS, stop_at_token=end_token)
references = [seq[1:] for seq in targets]
bleu = utils.calc_bleu_many(tokens, references)
sum_bleu += bleu
seq_count += 1
log.info("Processed %d phrases, mean BLEU = %.4f", seq_count, sum_bleu / seq_count)
