def __init__(self, ckpt_path='./Res/PretrainModel/2019_12_27_08_48_21/'):
checkpoint = Checkpoint.load(ckpt_path)
self.seq2seq = checkpoint.model
self.input_vocab = checkpoint.input_vocab
self.output_vocab = checkpoint.output_vocab
self.predictor = Predictor(self.seq2seq, self.input_vocab,
self.output_vocab)
def dev_predict(task_path, src_str, is_plot=True):
"""Helper used to visualize and understand why and what the model predicts.
Args:
task_path (str): path to the saved task directory containing, amongst
other, the model.
src_str (str): source sentence that will be used to predict.
is_plot (bool, optional): whether to plots the attention pattern.
Returns:
out_words (list): decoder predictions.
other (dictionary): additional information used for predictions.
test (dictionary): additional information that is only stored in dev mode.
These can include temporary variables that do not have to be stored in
`other` but that can still be interesting to inspect.
"""
check = Checkpoint.load(task_path)
check.model.set_dev_mode()
predictor = Predictor(check.model, check.input_vocab, check.output_vocab)
out_words, other = predictor.predict(src_str.split())
test = dict()
for k, v in other["test"].items():
tensor = v if isinstance(v, torch.Tensor) else torch.cat(v)
test[k] = tensor.detach().cpu().numpy().squeeze()[:other["length"][0]]
# except: # for using "step"
# test[k] = v
if is_plot:
visualizer = AttentionVisualizer(task_path)
visualizer(src_str)
return out_words, other, test
def test_predict(self):
predictor = Predictor(self.seq2seq,
self.dataset.input_vocab, self.dataset.output_vocab)
src_seq = ["I", "am", "fat"]
tgt_seq = predictor.predict(src_seq)
for tok in tgt_seq:
self.assertTrue(tok in self.dataset.output_vocab._token2index)
def predict_with_checkpoint(checkpoint_path,
sequence,
hierarchial = False,
remote = None,
word_vectors = None):
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = None, None
if word_vectors != None:
input_vects = Word2Vectors(input_vocab, word_vectors, word_vectors.dim_size)
output_vects = Word2Vectors(output_vocab, word_vectors, word_vectors.dim_size)
seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = input_vects, output_vects
seq2seq.decoder = TopKDecoder(seq2seq.decoder, 5)
if not hierarchial:
predictor = Predictor(seq2seq, input_vocab, output_vocab)
seq = sequence.strip().split()
else:
predictor = HierarchialPredictor(seq2seq, input_vocab, output_vocab)
seq = ['|'.join(x.split()) for x in sequence]
return ' '.join(predictor.predict(seq))
def predict(expt_dir, seq_str, date, epoch, step, n=3):
seq = seq_str.strip().split()
checkpoint_path = os.path.join(expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, date, epoch, step)
seq2seq, input_vocab, output_vocab = get_model(checkpoint_path)
beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4))
predictor = Predictor(beam_search, input_vocab, output_vocab)
return predictor.predict_n(seq, n=n)
def setUpClass(self):
test_path = os.path.dirname(os.path.realpath(__file__))
src = SourceField()
trg = TargetField()
dataset = torchtext.data.TabularDataset(
path=os.path.join(test_path, 'data/eng-fra.txt'), format='tsv',
fields=[('src', src), ('trg', trg)],
)
src.build_vocab(dataset)
trg.build_vocab(dataset)
encoder = EncoderRNN(len(src.vocab), 10, 10, rnn_cell='lstm')
decoder = DecoderRNN(len(trg.vocab), 10, 10, trg.sos_id, trg.eos_id, rnn_cell='lstm')
seq2seq = Seq2seq(encoder, decoder)
self.predictor = Predictor(seq2seq, src.vocab, trg.vocab)
class TestPredictor(unittest.TestCase):
@classmethod
def setUpClass(self):
test_path = os.path.dirname(os.path.realpath(__file__))
src = SourceField()
trg = TargetField()
dataset = torchtext.data.TabularDataset(
path=os.path.join(test_path, 'data/eng-fra.txt'),
format='tsv',
fields=[('src', src), ('trg', trg)],
)
src.build_vocab(dataset)
trg.build_vocab(dataset)
encoder = EncoderRNN(len(src.vocab), 10, 10, rnn_cell='lstm')
decoder = DecoderRNN(len(trg.vocab),
10,
10,
trg.sos_id,
trg.eos_id,
rnn_cell='lstm')
seq2seq = Seq2seq(encoder, decoder)
self.predictor = Predictor(seq2seq, src.vocab, trg.vocab)
def test_predict(self):
src_seq = "I am fat"
tgt_seq = self.predictor.predict(src_seq.split(' '))
for tok in tgt_seq:
self.assertTrue(tok in self.predictor.tgt_vocab.stoi)
def evaluate_model(model, data, src_field, tgt_field, file_props={}):
predictor = Predictor(model, src_field.vocab, tgt_field.vocab)
data["pred_lemma"] = [
"".join(predictor.predict(list(e.word))[:-1])
for e in data.itertuples()
]
acc = 0
for word in data.itertuples():
acc += int(word.pred_lemma == word.lemma)
acc /= len(data.lemma)
EXPERIMENT.metric("Dev accuracy", acc)
data.to_csv("./dev_{}.csv".format("-".join("{}={}".format(k, v)
for k, v in file_props)))
EXPERIMENT.log("Incorrect predictions")
EXPERIMENT.log(
str(data[data["lemma"] != data["pred_lemma"]][[
"word", "lemma", "pred_lemma"
]]))
class eval_tool:
def __init__(self, ckpt_path='./Res/PretrainModel/2019_12_27_08_48_21/'):
checkpoint = Checkpoint.load(ckpt_path)
self.seq2seq = checkpoint.model
self.input_vocab = checkpoint.input_vocab
self.output_vocab = checkpoint.output_vocab
self.predictor = Predictor(self.seq2seq, self.input_vocab,
self.output_vocab)
def predict(self, input_str):
return self.predictor.predict(input_str.strip().split())
def test(expt_dir, checkpoint, test_file, output_file):
if checkpoint is not None:
checkpoint_path = os.path.join(expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
checkpoint)
logging.info("loading checkpoint from {}".format(checkpoint_path))
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
else:
raise Exception("checkpoint path does not exist")
predictor = Predictor(seq2seq, input_vocab, output_vocab)
output = open(output_file, 'ab')
with open(test_file) as f:
for line_ in f:
line = line_.strip().split('<s>')
if len(line) != 0:
question = basic_tokenizer(line[-2])
answer = predictor.predict(question)[:-1]
output.write(''.join(answer) + '\n')
def test(opt, test_path):
if opt.load_checkpoint is not None:
# load model
logging.info("loading check point from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
# Prepare predictor
predictor = Predictor(seq2seq, input_vocab, output_vocab)
with open(test_path, 'r') as reader, open(test_path + '_pred',
'w') as writer:
for line in reader:
source = treebank_tokenizer(line.split("\t")[0])
writer.write(generate(source, predictor) + '\n')
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss, batch_size=32,
checkpoint_every=50,
print_every=10, expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq, train,
num_epochs=40, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
predictor = Predictor(seq2seq, input_vocab, output_vocab)
#while True:
#seq_str = raw_input("Type in a source sequence:")
#seq = seq_str.strip().split()
#print(predictor.predict(seq))
with open(opt.test_path) as f:
content = f.readlines()
content = [x.strip() for x in content]
output = ""
for row in content:
seq_in = row.split("\t")[0]
seq_out = row.split("\t")[1]
seq_pred = predictor.predict(seq_in.strip().split())
seq_pred = " ".join(seq_pred[:-1])
best_model_dir=opt.best_model_dir,
batch_size=opt.batch_size,
checkpoint_every=opt.checkpoint_every,
print_every=opt.print_every,
max_epochs=opt.max_epochs,
max_steps=opt.max_steps,
max_checkpoints_num=opt.max_checkpoints_num,
best_ppl=opt.best_ppl,
device=device,
multi_gpu=multi_gpu,
logger=logger)
seq2seq = t.train(seq2seq,
data=train,
start_step=opt.skip_steps,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=opt.teacher_forcing_ratio)
elif opt.phase == "infer":
# Predict
predictor = Predictor(seq2seq, src_vocab.word2idx, tgt_vocab.idx2word,
device)
while True:
seq_str = input("Type in a source sequence:")
seq = seq_str.strip().split()
ans = predictor.predict_n(seq, n=opt.beam_width) \
if opt.beam_width > 1 else predictor.predict(seq)
print(ans)
# train
t = SupervisedTrainer(
loss=loss,
batch_size=32,
checkpoint_every=50,
print_every=10,
expt_dir=opt.expt_dir,
)
seq2seq = t.train(
seq2seq,
train,
num_epochs=6,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume,
)
evaluator = Evaluator(loss=loss, batch_size=32)
dev_loss, accuracy = evaluator.evaluate(seq2seq, dev)
assert dev_loss < 1.5
beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 3))
predictor = Predictor(beam_search, input_vocab, output_vocab)
inp_seq = "1 3 5 7 9"
seq = predictor.predict(inp_seq.split())
assert " ".join(seq[:-1]) == inp_seq[::-1]
if par_in_list == '(':
word_list.append(')')
elif par_in_list == '[':
word_list.append(']')
elif par_in_list == '{':
word_list.append('}')
word_list = [word for word in word_list if word != '']
return ' '.join(word_list)
# In[22]:
predictor = Predictor(seq2seq_model, input_vocab, output_vocab)
num_samples = 0
perfect_samples = 0
dfa_perfect_samples = 0
match = 0
total = 0
model_correct = 0
model_wrong = 0
with torch.no_grad():
for batch in batch_iterator:
num_samples = num_samples + 1
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss,
batch_size=10,
checkpoint_every=10,
print_every=10,
expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq,
train,
num_epochs=1,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
if __name__ == '__main__':
predictor = Predictor(seq2seq, input_vocab, output_vocab)
# while True:
# seq_str = raw_input("Type in a source sequence:")
# seq = seq_str.strip().split()
# print(predictor.predict(seq))
def train():
src = SourceField(sequential=True,
tokenize=lambda x: [i for i in jieba.lcut(x)])
tgt = TargetField(sequential=True,
tokenize=lambda x: [i for i in jieba.lcut(x)])
max_len = 50
def len_filter(example):
return len(example.src) <= max_len and len(example.tgt) <= max_len
train = torchtext.data.TabularDataset(path=opt.train_path,
format='csv',
fields=[('src', src), ('tgt', tgt)],
filter_pred=len_filter)
dev = torchtext.data.TabularDataset(path=opt.dev_path,
format='csv',
fields=[('src', src), ('tgt', tgt)],
filter_pred=len_filter)
src.build_vocab(train, max_size=50000)
tgt.build_vocab(train, max_size=50000)
input_vocab = src.vocab
output_vocab = tgt.vocab
# NOTE: If the source field name and the target field name
# are different from 'src' and 'tgt' respectively, they have
# to be set explicitly before any training or inference
# seq2seq.src_field_name = 'src'
# seq2seq.tgt_field_name = 'tgt'
# Prepare loss
weight = torch.ones(len(tgt.vocab))
pad = tgt.vocab.stoi[tgt.pad_token]
loss = Perplexity(weight, pad)
if torch.cuda.is_available():
loss.cuda()
seq2seq = None
optimizer = None
if not opt.resume:
# Initialize model
hidden_size = 128
bidirectional = True
encoder = EncoderRNN(len(src.vocab),
max_len,
hidden_size,
bidirectional=bidirectional,
variable_lengths=True)
decoder = DecoderRNN(len(tgt.vocab),
max_len,
hidden_size * 2 if bidirectional else hidden_size,
dropout_p=0.2,
use_attention=True,
bidirectional=bidirectional,
eos_id=tgt.eos_id,
sos_id=tgt.sos_id)
seq2seq = Seq2seq(encoder, decoder)
if torch.cuda.is_available():
seq2seq.cuda()
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss,
batch_size=32,
checkpoint_every=50,
print_every=10,
expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq,
train,
num_epochs=6,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
predictor = Predictor(seq2seq, input_vocab, output_vocab)
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss,
batch_size=batch_size,
checkpoint_every=50,
print_every=10,
expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq,
train,
num_epochs=num_epochs,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
predictor = Predictor(seq2seq, input_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq_1 = [first_field.SYM_SOS
] + seq_str.strip().split() + [first_field.SYM_EOS]
seq_str = raw_input("Type in a source sequence:")
seq_2 = [first_field.SYM_SOS
] + seq_str.strip().split() + [first_field.SYM_EOS]
print(predictor.predict([seq_1, seq_2]))
parser = argparse.ArgumentParser()
parser.add_argument('--input',
action='store',
dest='input',
help='Input file',
default='INPUT_FILE')
opt = parser.parse_args()
checkpoint_path = os.path.join(ROOT, "experiment/checkpoints/72")
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
predictor = Predictor(seq2seq, input_vocab, output_vocab)
data = pd.read_csv(ROOT + INPUT_FILE, delimiter=",", header=0)
pred = []
def sentence_gen(sen):
#sen[max_len,batch_size]
a = []
for i in range(len(sen)):
phrase = ""
for j in range(len(sen[i])):
if sen[i][j] != "<eos>":
#print("printing word :",sen[i][j])
if sen[i][j] == "." or sen[i][j] == ",":
class AttentionVisualizer(object):
"""Object for visualizing the attention pattern of a given prediction.
Args:
task_path (str): name of the checkpoint file.
figsizeh (tuple, optional): (width, height) of the final matplotlib figure.
decimals (int, optional): number of decimals to whoe when pritning any number.
is_show_attn_split (bool, optional): whether to show the the content and
positional attention if there is one in addition to the full attention.
is_show_evaluation (bool, optional): whether to show the evaluation metric
if the target is given.
output_length_key, attention_key, content_attn_key (str, optional): keys of
the respective values in the the dictionary returned by the prediction.
positional_table_labels (dictionary, optional): mapping from the keys
in the return dictionary (the values) to the name the name of it should
be shown as in the figure (the keys). The order is the one that will
be used to plot the table (in python > 3.6).
is_show_name (bool, optional): whether to show the name of the mdoel as
the title of the figure.
max_src, max_out, max_tgt (int, optional): maximum number of token to show
for the source, the output and the target. Used in order not to clotter
too much the plots.
kwargs:
Additional arguments to `MetricComputer`.
"""
def __init__(
self,
task_path,
figsize=(15, 13),
decimals=2,
is_show_attn_split=True,
is_show_evaluation=True,
output_length_key='length',
attention_key="attention_score",
position_attn_key='position_attention',
content_attn_key='content_attention',
positional_table_labels={
"λ%": "position_percentage",
"C.γ": "content_confidence",
#"lgt": "approx_max_logit",
"C.λ": "pos_confidence",
"μ": "mu",
"σ": "sigma",
"w_α": "mean_attn_old_weight",
"w_j/n": "rel_counter_decoder_weight",
"w_1/n": "single_step_weight",
"w_μ": "mu_old_weight",
"w_γ": "mean_content_old_weight",
"w_1": "bias_weight"
},
# "% carry": "carry_rates",
is_show_name=True,
max_src=17,
max_out=13,
max_tgt=13,
**kwargs):
check = Checkpoint.load(task_path)
self.model = check.model
# store some interesting variables
self.model.set_dev_mode()
self.predictor = Predictor(self.model, check.input_vocab,
check.output_vocab)
self.model_name = task_path.split("/")[-2]
self.figsize = figsize
self.decimals = decimals
self.is_show_attn_split = is_show_attn_split
self.is_show_evaluation = is_show_evaluation
self.positional_table_labels = positional_table_labels
self.is_show_name = is_show_name
self.max_src = max_src
self.max_out = max_out
self.max_tgt = max_tgt
self.output_length_key = output_length_key
self.attention_key = attention_key
self.position_attn_key = position_attn_key
self.content_attn_key = content_attn_key
if self.is_show_evaluation:
self.is_symbol_rewriting = "symbol rewriting" in task_path.lower()
self.metric_computer = MetricComputer(
check, is_symbol_rewriting=self.is_symbol_rewriting, **kwargs)
if self.model.decoder.is_attention is None:
raise AttentionException("Model is not using attention.")
def __call__(self, src_str, tgt_str=None):
"""Plots the attention for the current example.
Args:
src_str (str): source of the example.
tgt_str (str, optional): (width, height) target of the example,
must be given in order to show the final metric.
Returns:
fig (plt.Figure): plotted attention figure.
"""
out_words, other = self.predictor.predict(src_str.split())
full_src_str = src_str
full_out_str = " ".join(out_words)
full_tgt_str = tgt_str
additional, additional_text = self._format_additional(other)
additional, src_words, out_words, tgt_str = self._subset(
additional, src_str.split(), out_words, tgt_str)
if self.is_show_name:
title = ""
else:
title = None
if tgt_str is not None:
if self.is_show_name:
title += "\n tgt_str: {} - ".format(tgt_str)
else:
title = "tgt_str: {} - ".format(tgt_str)
if self.metric_computer.is_predict_eos:
is_output_good_length = (len(full_out_str.split()) != len(
full_tgt_str.split()))
if self.is_symbol_rewriting and is_output_good_length:
warnings.warn(
"Cannot currently show the metric for symbol rewriting if output is not the right length."
)
else:
metrics = self.metric_computer(full_src_str, full_out_str,
full_tgt_str)
for name, val in metrics.items():
title += "{}: {:.2g} ".format(name, val)
else:
warnings.warn(
"Cannot currently show the metric in the attention plots when `is_predict_eos=False`"
)
if self.attention_key not in additional:
raise ValueError(
"`{}` not returned by predictor. Make sure the model uses attention."
.format(self.attention_key))
attention = additional[self.attention_key]
if self.position_attn_key in additional:
filtered_pos_table_labels = {
k: v
for k, v in self.positional_table_labels.items()
if v in additional
}
table_values = np.stack([
np.around(additional[name], decimals=self.decimals)
for name in filtered_pos_table_labels.values()
]).T
if self.is_show_attn_split and (self.position_attn_key in additional
and self.content_attn_key
in additional):
content_attention = additional.get(self.content_attn_key)
positional_attention = additional.get(self.position_attn_key)
fig, axs = plt.subplots(2, 2, figsize=self.figsize)
_plot_attention(src_words,
out_words,
attention,
axs[0, 0],
is_colorbar=False,
title="Final Attention")
_plot_table(table_values, list(filtered_pos_table_labels.keys()),
axs[0, 1])
_plot_attention(src_words,
out_words,
content_attention,
axs[1, 0],
title="Content Attention")
_plot_attention(src_words,
out_words,
positional_attention,
axs[1, 1],
title="Positional Attention")
elif self.position_attn_key in additional:
fig, axs = plt.subplots(1, 2, figsize=self.figsize)
_plot_attention(src_words,
out_words,
attention,
axs[0],
title="Final Attention")
_plot_table(table_values, list(filtered_pos_table_labels.keys()),
axs[1])
else:
fig, ax = plt.subplots(1, 1, figsize=self.figsize)
_plot_attention(src_words,
out_words,
attention,
ax,
title="Final Attention")
fig.text(0.5,
0.02,
' | '.join(additional_text),
ha='center',
va='center',
size=13)
if title is not None:
plt.suptitle(title, size=13, weight="bold")
fig.tight_layout()
fig.subplots_adjust(bottom=0.07, top=0.83)
return fig
def _format_additional(self, additional):
"""Format the additinal dictionary returned by the predictor."""
def _format_carry_rates(carry_rates):
if carry_rates is None:
return "Carry % : None"
mean_carry_rates = np.around(carry_rates.mean().item(),
decimals=self.decimals)
median_carry_rates = np.around(carry_rates.median().item(),
decimals=self.decimals)
return "Carry % : mean: {}; median: {}".format(
mean_carry_rates, median_carry_rates)
def _format_bb_gates(gates):
if gates is None:
return "BB Weight Mean Gates : None"
mean_gates = np.around(gates.mean(0), decimals=self.decimals)
return "BB Weight Mean Gates : {}".format(mean_gates)
def _format_mu_weights(mu_weights):
if mu_weights is not None:
building_blocks_labels = self.model.decoder.position_attention.bb_labels
for i, label in enumerate(building_blocks_labels):
output[label + "_weight"] = mu_weights[:, i]
output = dict()
additional.pop(
"visualize",
None) # this is only for training visualization not predict
additional.pop("losses", None)
additional_text = []
additional = flatten_dict(additional)
output = dict()
output[self.output_length_key] = additional.pop(
self.output_length_key)[0]
for k, v in additional.items():
tensor = v if isinstance(v, torch.Tensor) else torch.cat(v)
output[k] = tensor.detach().cpu().numpy().squeeze(
)[:output[self.output_length_key]]
carry_txt = _format_carry_rates(additional.pop("carry_rates", None))
bb_gates_txt = _format_bb_gates(output.pop("bb_gates", None))
additional_text.append(carry_txt)
additional_text.append(bb_gates_txt)
_format_mu_weights(output.pop("mu_weights", None))
return output, additional_text
def _subset(self, additional, src_words, out_words, tgt_str=None):
"""Subsets the objects in the additional dictionary in order not to
clotter the visualization.
"""
n_src = len(src_words)
n_out = len(out_words)
if n_out > self.max_out:
subset_out = self.max_out // 2
out_words = out_words[:subset_out] + out_words[-subset_out:]
for k, v in additional.items():
if isinstance(v, np.ndarray):
additional[k] = np.concatenate(
(v[:subset_out], v[-subset_out:]), axis=0)
if n_src > self.max_src:
subset_src = self.max_src // 2
src_words = src_words[:subset_src] + src_words[-subset_src:]
for k, v in additional.items():
if isinstance(v, np.ndarray) and v.ndim == 2:
additional[k] = np.concatenate(
(v[:, :subset_src], v[:, -subset_src:]), axis=1)
if tgt_str is not None:
tgt_words = tgt_str.split()
n_tgt = len(tgt_words)
if n_tgt > self.max_tgt:
subset_target = self.max_tgt // 2
tgt_str = " ".join(tgt_words[:subset_target] + ["..."] +
tgt_words[-subset_target:])
return additional, src_words, out_words, tgt_str
parser.add_argument('--log-level',
dest='log_level',
default='info',
help='Logging level.')
opt = parser.parse_args()
LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
logging.basicConfig(format=LOG_FORMAT,
level=getattr(logging, opt.log_level.upper()))
logging.info(opt)
if opt.load_checkpoint is not None:
logging.info("loading checkpoint from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
predictor = Predictor(seq2seq, input_vocab, output_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
max_grad_norm=args.max_grad_norm)
############### train model ################
t = SpkTrainer(args=args,
loss=loss,
batch_size=args.batch_size,
checkpoint_every=args.ckpt_every,
random_seed=args.seed,
print_every=args.verbose,
expt_dir=args.expt_dir)
discrim = t.train(model=model,
data=train,
num_epochs=args.epochs,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=args.resume)
################ initialize predictor ###################
predictor = Predictor(model=model,
src_vocab=input_vocab,
tgt_vocab=output_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
dropout_p=0.2, use_attention=True,
bidirectional=bidirectional,
rnn_cell='lstm',
eos_id=tgt.eos_id, sos_id=tgt.sos_id)
seq2seq = Seq2seq(encoder, decoder)
if torch.cuda.is_available():
seq2seq.cuda()
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# train
t = SupervisedTrainer(loss=loss, batch_size=32,
checkpoint_every=50,
print_every=10, expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq, train,
num_epochs=6, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
evaluator = Evaluator(loss=loss, batch_size=32)
dev_loss, accuracy = evaluator.evaluate(seq2seq, dev)
assert dev_loss < 1.5
predictor = Predictor(seq2seq, input_vocab, output_vocab)
inp_seq = "1 3 5 7 9"
seq = predictor.predict(inp_seq.split())
assert " ".join(seq[:-1]) == inp_seq[::-1]
for param in seq2tree.parameters():
param.data.uniform_(-0.08, 0.08)
# encoder.embedding.weight.data.set_(input_vocab.vectors)
# encoder.embedding.weight.data.set_(output_vocab.vectors)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
optimizer = Optimizer(optim.Adam(seq2tree.parameters(), lr=1e-4), max_grad_norm=5)
# train
t = SupervisedTrainer(loss=loss, batch_size=1,
checkpoint_every=50,
print_every=10, expt_dir=opt.expt_dir)
seq2tree = t.train(seq2tree, train,
num_epochs=20, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
predictor = Predictor(seq2tree, input_vocab, input_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
# TODO add dev eval here for early stopping
if config['train model']:
seq2seq = t.train(input_vocab,
feats_vocab,
seq2seq,
train,
num_epochs=config['epochs'],
vectors=vectors,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
# pdb.set_trace()
predictor = Predictor(seq2seq, input_vocab, feats_vocab, output_vocab, vectors)
# seq2top = Seq2seq(seq2seq.encoder, )
# topk_predictor = Predictor(seq2top, input_vocab, output_vocab, vectors)
if config['pull embeddings']:
out_vecs = {}
if config['feat embeddings']:
feats = {}
of = open(config['feat output'], 'wb')
# TODO add option to save output
src = SourceField()
feat = SourceField()
tgt = TargetField()
# pdb.set_trace()
def sample(
# train_source,
# train_target,
# dev_source,
# dev_target,
experiment_directory='/home/xweiwang/RL/seq2seq/experiment',
checkpoint='2019_05_18_20_32_54',
resume=True,
log_level='info',
):
"""
# Sample usage
TRAIN_SRC=data/toy_reverse/train/src.txt
TRAIN_TGT=data/toy_reverse/train/tgt.txt
DEV_SRC=data/toy_reverse/dev/src.txt
DEV_TGT=data/toy_reverse/dev/tgt.txt
## Training
```shell
$ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt
$EXPT_PATH
```
## Resuming from the latest checkpoint of the experiment
```shell
$ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt
$EXPT_PATH -r
```
## Resuming from a specific checkpoint
```shell
$ python examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt
$EXPT_PATH -c $CHECKPOINT_DIR
```
"""
logging.basicConfig(
format=LOG_FORMAT,
level=getattr(logging, log_level.upper()),
)
# logging.info('train_source: %s', train_source)
# logging.info('train_target: %s', train_target)
# logging.info('dev_source: %s', dev_source)
# logging.info('dev_target: %s', dev_target)
logging.info('experiment_directory: %s', experiment_directory)
logging.info('checkpoint: %s', checkpoint)
# if checkpoint:
seq2seq, input_vocab, output_vocab = load_checkpoint(
experiment_directory, checkpoint)
# else:
# seq2seq, input_vocab, output_vocab = train_model(
# train_source,
# train_target,
# dev_source,
# dev_target,
# experiment_directory,
# resume=resume,
# )
predictor = Predictor(seq2seq, input_vocab, output_vocab)
while True:
seq_str = input('Type in a source sequence: ')
seq = seq_str.strip().split()
print(predictor.predict(seq))
def run_training(opt, default_data_dir, num_epochs=100):
if opt.load_checkpoint is not None:
logging.info("loading checkpoint from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
else:
# Prepare dataset
src = SourceField()
tgt = TargetField()
max_len = 50
data_file = os.path.join(default_data_dir, opt.train_path, 'data.txt')
logging.info("Starting new Training session on %s", data_file)
def len_filter(example):
return (len(example.src) <= max_len) and (len(example.tgt) <= max_len) \
and (len(example.src) > 0) and (len(example.tgt) > 0)
train = torchtext.data.TabularDataset(
path=data_file, format='json',
fields={'src': ('src', src), 'tgt': ('tgt', tgt)},
filter_pred=len_filter
)
dev = None
if opt.no_dev is False:
dev_data_file = os.path.join(default_data_dir, opt.train_path, 'dev-data.txt')
dev = torchtext.data.TabularDataset(
path=dev_data_file, format='json',
fields={'src': ('src', src), 'tgt': ('tgt', tgt)},
filter_pred=len_filter
)
src.build_vocab(train, max_size=50000)
tgt.build_vocab(train, max_size=50000)
input_vocab = src.vocab
output_vocab = tgt.vocab
# NOTE: If the source field name and the target field name
# are different from 'src' and 'tgt' respectively, they have
# to be set explicitly before any training or inference
# seq2seq.src_field_name = 'src'
# seq2seq.tgt_field_name = 'tgt'
# Prepare loss
weight = torch.ones(len(tgt.vocab))
pad = tgt.vocab.stoi[tgt.pad_token]
loss = Perplexity(weight, pad)
if torch.cuda.is_available():
logging.info("Yayyy We got CUDA!!!")
loss.cuda()
else:
logging.info("No cuda available device found running on cpu")
seq2seq = None
optimizer = None
if not opt.resume:
hidden_size = 128
decoder_hidden_size = hidden_size * 2
logging.info("EncoderRNN Hidden Size: %s", hidden_size)
logging.info("DecoderRNN Hidden Size: %s", decoder_hidden_size)
bidirectional = True
encoder = EncoderRNN(len(src.vocab), max_len, hidden_size,
bidirectional=bidirectional,
rnn_cell='lstm',
variable_lengths=True)
decoder = DecoderRNN(len(tgt.vocab), max_len, decoder_hidden_size,
dropout_p=0, use_attention=True,
bidirectional=bidirectional,
rnn_cell='lstm',
eos_id=tgt.eos_id, sos_id=tgt.sos_id)
seq2seq = Seq2seq(encoder, decoder)
if torch.cuda.is_available():
seq2seq.cuda()
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
scheduler = StepLR(optimizer.optimizer, 1)
optimizer.set_scheduler(scheduler)
# train
num_epochs = num_epochs
batch_size = 32
checkpoint_every = num_epochs / 10
print_every = num_epochs / 100
properties = dict(batch_size=batch_size,
checkpoint_every=checkpoint_every,
print_every=print_every, expt_dir=opt.expt_dir,
num_epochs=num_epochs,
teacher_forcing_ratio=0.5,
resume=opt.resume)
logging.info("Starting training with the following Properties %s", json.dumps(properties, indent=2))
t = SupervisedTrainer(loss=loss, batch_size=num_epochs,
checkpoint_every=checkpoint_every,
print_every=print_every, expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq, train,
num_epochs=num_epochs, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
evaluator = Evaluator(loss=loss, batch_size=batch_size)
if opt.no_dev is False:
dev_loss, accuracy = evaluator.evaluate(seq2seq, dev)
logging.info("Dev Loss: %s", dev_loss)
logging.info("Accuracy: %s", dev_loss)
beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4))
predictor = Predictor(beam_search, input_vocab, output_vocab)
while True:
try:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
results = predictor.predict_n(seq, n=3)
for i, res in enumerate(results):
print('option %s: %s\n', i + 1, res)
except KeyboardInterrupt:
logging.info("Bye Bye")
exit(0)
def main():
'''Main Function'''
parser = argparse.ArgumentParser(description='sum_file.py')
parser.add_argument('-model', required=True,
help='Path to model .pt file')
parser.add_argument('-src', required=True,
help='Source sequence to decode (one line per sequence)')
parser.add_argument('-vocab', required=True,
help='Source sequence to decode (one line per sequence)')
parser.add_argument('-output', default='pred.txt',
help="""Path to output the predictions (each line will
be the decoded sequence""")
parser.add_argument('-beam_size', type=int, default=5,
help='Beam size')
parser.add_argument('-batch_size', type=int, default=30,
help='Batch size')
parser.add_argument('-n_best', type=int, default=1,
help="""If verbose is set, will output the n_best
decoded sentences""")
parser.add_argument('-no_cuda', action='store_true')
opt = parser.parse_args()
opt.cuda = not opt.no_cuda
# Prepare DataLoader
preprocess_data = torch.load(opt.vocab)
preprocess_settings = preprocess_data['settings']
test_src_word_insts = read_instances_from_file(
opt.src,
preprocess_settings.max_word_seq_len,
preprocess_settings.keep_case,
preprocess_settings.mode)
test_src_insts = convert_instance_to_idx_seq(
test_src_word_insts, preprocess_data['dict']['src'])
# prepare model
device = torch.device('cuda' if opt.cuda else 'cpu')
checkpoint = torch.load(opt.model)
model_opt = checkpoint['settings']
model_opt.bidirectional = True
encoder = EncoderRNN(model_opt.src_vocab_size, model_opt.max_token_seq_len, model_opt.d_model,
bidirectional=model_opt.bidirectional, variable_lengths=True)
decoder = DecoderRNN(model_opt.tgt_vocab_size, model_opt.max_token_seq_len, model_opt.d_model * 2 if model_opt.bidirectional else model_opt.d_model,
n_layers=model_opt.n_layer, dropout_p=model_opt.dropout, use_attention=True, bidirectional=model_opt.bidirectional,
eos_id=Constants.BOS, sos_id=Constants.EOS)
model = Seq2seq(encoder, decoder).to(device)
model = nn.DataParallel(model) # using Dataparallel because training used
model.load_state_dict(checkpoint['model'])
print('[Info] Trained model state loaded.')
predictor = Predictor(model, preprocess_data['dict']['tgt'])
with open(opt.output, 'w') as f:
for src_seq in tqdm(test_src_insts, mininterval=2, desc=' - (Test)', leave=False):
pred_line = ' '.join(predictor.predict(src_seq))
f.write(pred_line + '\n')
print('[Info] Finished.')
def __init__(
self,
task_path,
figsize=(15, 13),
decimals=2,
is_show_attn_split=True,
is_show_evaluation=True,
output_length_key='length',
attention_key="attention_score",
position_attn_key='position_attention',
content_attn_key='content_attention',
positional_table_labels={
"λ%": "position_percentage",
"C.γ": "content_confidence",
#"lgt": "approx_max_logit",
"C.λ": "pos_confidence",
"μ": "mu",
"σ": "sigma",
"w_α": "mean_attn_old_weight",
"w_j/n": "rel_counter_decoder_weight",
"w_1/n": "single_step_weight",
"w_μ": "mu_old_weight",
"w_γ": "mean_content_old_weight",
"w_1": "bias_weight"
},
# "% carry": "carry_rates",
is_show_name=True,
max_src=17,
max_out=13,
max_tgt=13,
**kwargs):
check = Checkpoint.load(task_path)
self.model = check.model
# store some interesting variables
self.model.set_dev_mode()
self.predictor = Predictor(self.model, check.input_vocab,
check.output_vocab)
self.model_name = task_path.split("/")[-2]
self.figsize = figsize
self.decimals = decimals
self.is_show_attn_split = is_show_attn_split
self.is_show_evaluation = is_show_evaluation
self.positional_table_labels = positional_table_labels
self.is_show_name = is_show_name
self.max_src = max_src
self.max_out = max_out
self.max_tgt = max_tgt
self.output_length_key = output_length_key
self.attention_key = attention_key
self.position_attn_key = position_attn_key
self.content_attn_key = content_attn_key
if self.is_show_evaluation:
self.is_symbol_rewriting = "symbol rewriting" in task_path.lower()
self.metric_computer = MetricComputer(
check, is_symbol_rewriting=self.is_symbol_rewriting, **kwargs)
if self.model.decoder.is_attention is None:
raise AttentionException("Model is not using attention.")
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss, batch_size=32,
checkpoint_every=50,
print_every=10, expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq, train,
num_epochs=6, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
topk_decoder = seq2seq.decoder
topk_decoder = TopKDecoder(decoder, 10, False , True ,1.0)
seq2seq = Seq2seq(encoder, topk_decoder)
predictor = Predictor(seq2seq, input_vocab, output_vocab, False,1,float('inf'))
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))