def __init__(self, ckpt_path='./Res/PretrainModel/2019_12_27_08_48_21/'):
checkpoint = Checkpoint.load(ckpt_path)
self.seq2seq = checkpoint.model
self.input_vocab = checkpoint.input_vocab
self.output_vocab = checkpoint.output_vocab
self.predictor = Predictor(self.seq2seq, self.input_vocab,
self.output_vocab)
def predict_with_checkpoint(checkpoint_path,
sequence,
hierarchial = False,
remote = None,
word_vectors = None):
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = None, None
if word_vectors != None:
input_vects = Word2Vectors(input_vocab, word_vectors, word_vectors.dim_size)
output_vects = Word2Vectors(output_vocab, word_vectors, word_vectors.dim_size)
seq2seq.encoder.word_vectors, seq2seq.decoder.word_vectors = input_vects, output_vects
seq2seq.decoder = TopKDecoder(seq2seq.decoder, 5)
if not hierarchial:
predictor = Predictor(seq2seq, input_vocab, output_vocab)
seq = sequence.strip().split()
else:
predictor = HierarchialPredictor(seq2seq, input_vocab, output_vocab)
seq = ['|'.join(x.split()) for x in sequence]
return ' '.join(predictor.predict(seq))
def predict(expt_dir, seq_str, date, epoch, step, n=3):
seq = seq_str.strip().split()
checkpoint_path = os.path.join(expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, date, epoch, step)
seq2seq, input_vocab, output_vocab = get_model(checkpoint_path)
beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4))
predictor = Predictor(beam_search, input_vocab, output_vocab)
return predictor.predict_n(seq, n=n)
def test_predict(self):
predictor = Predictor(self.seq2seq,
self.dataset.input_vocab, self.dataset.output_vocab)
src_seq = ["I", "am", "fat"]
tgt_seq = predictor.predict(src_seq)
for tok in tgt_seq:
self.assertTrue(tok in self.dataset.output_vocab._token2index)
def dev_predict(task_path, src_str, is_plot=True):
"""Helper used to visualize and understand why and what the model predicts.
Args:
task_path (str): path to the saved task directory containing, amongst
other, the model.
src_str (str): source sentence that will be used to predict.
is_plot (bool, optional): whether to plots the attention pattern.
Returns:
out_words (list): decoder predictions.
other (dictionary): additional information used for predictions.
test (dictionary): additional information that is only stored in dev mode.
These can include temporary variables that do not have to be stored in
`other` but that can still be interesting to inspect.
"""
check = Checkpoint.load(task_path)
check.model.set_dev_mode()
predictor = Predictor(check.model, check.input_vocab, check.output_vocab)
out_words, other = predictor.predict(src_str.split())
test = dict()
for k, v in other["test"].items():
tensor = v if isinstance(v, torch.Tensor) else torch.cat(v)
test[k] = tensor.detach().cpu().numpy().squeeze()[:other["length"][0]]
# except: # for using "step"
# test[k] = v
if is_plot:
visualizer = AttentionVisualizer(task_path)
visualizer(src_str)
return out_words, other, test
def setUpClass(self):
test_path = os.path.dirname(os.path.realpath(__file__))
src = SourceField()
trg = TargetField()
dataset = torchtext.data.TabularDataset(
path=os.path.join(test_path, 'data/eng-fra.txt'), format='tsv',
fields=[('src', src), ('trg', trg)],
)
src.build_vocab(dataset)
trg.build_vocab(dataset)
encoder = EncoderRNN(len(src.vocab), 10, 10, rnn_cell='lstm')
decoder = DecoderRNN(len(trg.vocab), 10, 10, trg.sos_id, trg.eos_id, rnn_cell='lstm')
seq2seq = Seq2seq(encoder, decoder)
self.predictor = Predictor(seq2seq, src.vocab, trg.vocab)
def evaluate_model(model, data, src_field, tgt_field, file_props={}):
predictor = Predictor(model, src_field.vocab, tgt_field.vocab)
data["pred_lemma"] = [
"".join(predictor.predict(list(e.word))[:-1])
for e in data.itertuples()
]
acc = 0
for word in data.itertuples():
acc += int(word.pred_lemma == word.lemma)
acc /= len(data.lemma)
EXPERIMENT.metric("Dev accuracy", acc)
data.to_csv("./dev_{}.csv".format("-".join("{}={}".format(k, v)
for k, v in file_props)))
EXPERIMENT.log("Incorrect predictions")
EXPERIMENT.log(
str(data[data["lemma"] != data["pred_lemma"]][[
"word", "lemma", "pred_lemma"
]]))
def test(opt, test_path):
if opt.load_checkpoint is not None:
# load model
logging.info("loading check point from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
# Prepare predictor
predictor = Predictor(seq2seq, input_vocab, output_vocab)
with open(test_path, 'r') as reader, open(test_path + '_pred',
'w') as writer:
for line in reader:
source = treebank_tokenizer(line.split("\t")[0])
writer.write(generate(source, predictor) + '\n')
def test(expt_dir, checkpoint, test_file, output_file):
if checkpoint is not None:
checkpoint_path = os.path.join(expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
checkpoint)
logging.info("loading checkpoint from {}".format(checkpoint_path))
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
else:
raise Exception("checkpoint path does not exist")
predictor = Predictor(seq2seq, input_vocab, output_vocab)
output = open(output_file, 'ab')
with open(test_file) as f:
for line_ in f:
line = line_.strip().split('<s>')
if len(line) != 0:
question = basic_tokenizer(line[-2])
answer = predictor.predict(question)[:-1]
output.write(''.join(answer) + '\n')
def train():
src = SourceField(sequential=True,
tokenize=lambda x: [i for i in jieba.lcut(x)])
tgt = TargetField(sequential=True,
tokenize=lambda x: [i for i in jieba.lcut(x)])
max_len = 50
def len_filter(example):
return len(example.src) <= max_len and len(example.tgt) <= max_len
train = torchtext.data.TabularDataset(path=opt.train_path,
format='csv',
fields=[('src', src), ('tgt', tgt)],
filter_pred=len_filter)
dev = torchtext.data.TabularDataset(path=opt.dev_path,
format='csv',
fields=[('src', src), ('tgt', tgt)],
filter_pred=len_filter)
src.build_vocab(train, max_size=50000)
tgt.build_vocab(train, max_size=50000)
input_vocab = src.vocab
output_vocab = tgt.vocab
# NOTE: If the source field name and the target field name
# are different from 'src' and 'tgt' respectively, they have
# to be set explicitly before any training or inference
# seq2seq.src_field_name = 'src'
# seq2seq.tgt_field_name = 'tgt'
# Prepare loss
weight = torch.ones(len(tgt.vocab))
pad = tgt.vocab.stoi[tgt.pad_token]
loss = Perplexity(weight, pad)
if torch.cuda.is_available():
loss.cuda()
seq2seq = None
optimizer = None
if not opt.resume:
# Initialize model
hidden_size = 128
bidirectional = True
encoder = EncoderRNN(len(src.vocab),
max_len,
hidden_size,
bidirectional=bidirectional,
variable_lengths=True)
decoder = DecoderRNN(len(tgt.vocab),
max_len,
hidden_size * 2 if bidirectional else hidden_size,
dropout_p=0.2,
use_attention=True,
bidirectional=bidirectional,
eos_id=tgt.eos_id,
sos_id=tgt.sos_id)
seq2seq = Seq2seq(encoder, decoder)
if torch.cuda.is_available():
seq2seq.cuda()
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss,
batch_size=32,
checkpoint_every=50,
print_every=10,
expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq,
train,
num_epochs=6,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
predictor = Predictor(seq2seq, input_vocab, output_vocab)
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss, batch_size=32,
checkpoint_every=50,
print_every=10, expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq, train,
num_epochs=6, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
topk_decoder = seq2seq.decoder
topk_decoder = TopKDecoder(decoder, 10, False , True ,1.0)
seq2seq = Seq2seq(encoder, topk_decoder)
predictor = Predictor(seq2seq, input_vocab, output_vocab, False,1,float('inf'))
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
def main():
'''Main Function'''
parser = argparse.ArgumentParser(description='sum_file.py')
parser.add_argument('-model', required=True,
help='Path to model .pt file')
parser.add_argument('-src', required=True,
help='Source sequence to decode (one line per sequence)')
parser.add_argument('-vocab', required=True,
help='Source sequence to decode (one line per sequence)')
parser.add_argument('-output', default='pred.txt',
help="""Path to output the predictions (each line will
be the decoded sequence""")
parser.add_argument('-beam_size', type=int, default=5,
help='Beam size')
parser.add_argument('-batch_size', type=int, default=30,
help='Batch size')
parser.add_argument('-n_best', type=int, default=1,
help="""If verbose is set, will output the n_best
decoded sentences""")
parser.add_argument('-no_cuda', action='store_true')
opt = parser.parse_args()
opt.cuda = not opt.no_cuda
# Prepare DataLoader
preprocess_data = torch.load(opt.vocab)
preprocess_settings = preprocess_data['settings']
test_src_word_insts = read_instances_from_file(
opt.src,
preprocess_settings.max_word_seq_len,
preprocess_settings.keep_case,
preprocess_settings.mode)
test_src_insts = convert_instance_to_idx_seq(
test_src_word_insts, preprocess_data['dict']['src'])
# prepare model
device = torch.device('cuda' if opt.cuda else 'cpu')
checkpoint = torch.load(opt.model)
model_opt = checkpoint['settings']
model_opt.bidirectional = True
encoder = EncoderRNN(model_opt.src_vocab_size, model_opt.max_token_seq_len, model_opt.d_model,
bidirectional=model_opt.bidirectional, variable_lengths=True)
decoder = DecoderRNN(model_opt.tgt_vocab_size, model_opt.max_token_seq_len, model_opt.d_model * 2 if model_opt.bidirectional else model_opt.d_model,
n_layers=model_opt.n_layer, dropout_p=model_opt.dropout, use_attention=True, bidirectional=model_opt.bidirectional,
eos_id=Constants.BOS, sos_id=Constants.EOS)
model = Seq2seq(encoder, decoder).to(device)
model = nn.DataParallel(model) # using Dataparallel because training used
model.load_state_dict(checkpoint['model'])
print('[Info] Trained model state loaded.')
predictor = Predictor(model, preprocess_data['dict']['tgt'])
with open(opt.output, 'w') as f:
for src_seq in tqdm(test_src_insts, mininterval=2, desc=' - (Test)', leave=False):
pred_line = ' '.join(predictor.predict(src_seq))
f.write(pred_line + '\n')
print('[Info] Finished.')
def sample(
# train_source,
# train_target,
# dev_source,
# dev_target,
experiment_directory='/home/xweiwang/RL/seq2seq/experiment',
checkpoint='2019_05_18_20_32_54',
resume=True,
log_level='info',
):
"""
# Sample usage
TRAIN_SRC=data/toy_reverse/train/src.txt
TRAIN_TGT=data/toy_reverse/train/tgt.txt
DEV_SRC=data/toy_reverse/dev/src.txt
DEV_TGT=data/toy_reverse/dev/tgt.txt
## Training
```shell
$ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt
$EXPT_PATH
```
## Resuming from the latest checkpoint of the experiment
```shell
$ ./examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt
$EXPT_PATH -r
```
## Resuming from a specific checkpoint
```shell
$ python examples/sample.py $TRAIN_SRC $TRAIN_TGT $DEV_SRC $DEV_TGT -expt
$EXPT_PATH -c $CHECKPOINT_DIR
```
"""
logging.basicConfig(
format=LOG_FORMAT,
level=getattr(logging, log_level.upper()),
)
# logging.info('train_source: %s', train_source)
# logging.info('train_target: %s', train_target)
# logging.info('dev_source: %s', dev_source)
# logging.info('dev_target: %s', dev_target)
logging.info('experiment_directory: %s', experiment_directory)
logging.info('checkpoint: %s', checkpoint)
# if checkpoint:
seq2seq, input_vocab, output_vocab = load_checkpoint(
experiment_directory, checkpoint)
# else:
# seq2seq, input_vocab, output_vocab = train_model(
# train_source,
# train_target,
# dev_source,
# dev_target,
# experiment_directory,
# resume=resume,
# )
predictor = Predictor(seq2seq, input_vocab, output_vocab)
while True:
seq_str = input('Type in a source sequence: ')
seq = seq_str.strip().split()
print(predictor.predict(seq))
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
# train
t = SupervisedTrainer(loss=loss,
batch_size=16,
checkpoint_every=50,
print_every=1,
expt_dir=opt.expt_dir)
model = t.train(gnn,
data_train,
data_valid,
num_epochs=30,
num_antecedents=opt.num_antecedents,
optimizer=optimizer,
teacher_forcing_ratio=1,
resume=opt.resume,
save_file=opt.save_file,
embed_file=opt.embed_file,
log_dir=opt.log_dir)
predictor = Predictor(model, input_vocab, output_vocab)
"""
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
"""
print("Training Finished.")
best_model_dir=opt.best_model_dir,
batch_size=opt.batch_size,
checkpoint_every=opt.checkpoint_every,
print_every=opt.print_every,
max_epochs=opt.max_epochs,
max_steps=opt.max_steps,
max_checkpoints_num=opt.max_checkpoints_num,
best_ppl=opt.best_ppl,
device=device,
multi_gpu=multi_gpu,
logger=logger)
seq2seq = t.train(seq2seq,
data=train,
start_step=opt.skip_steps,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=opt.teacher_forcing_ratio)
elif opt.phase == "infer":
# Predict
predictor = Predictor(seq2seq, src_vocab.word2idx, tgt_vocab.idx2word,
device)
while True:
seq_str = input("Type in a source sequence:")
seq = seq_str.strip().split()
ans = predictor.predict_n(seq, n=opt.beam_width) \
if opt.beam_width > 1 else predictor.predict(seq)
print(ans)
if par_in_list == '(':
word_list.append(')')
elif par_in_list == '[':
word_list.append(']')
elif par_in_list == '{':
word_list.append('}')
word_list = [word for word in word_list if word != '']
return ' '.join(word_list)
# In[22]:
predictor = Predictor(seq2seq_model, input_vocab, output_vocab)
num_samples = 0
perfect_samples = 0
dfa_perfect_samples = 0
match = 0
total = 0
model_correct = 0
model_wrong = 0
with torch.no_grad():
for batch in batch_iterator:
num_samples = num_samples + 1
def train(opt):
LOG_FORMAT = '%(asctime)s %(levelname)-8s %(message)s'
logging.basicConfig(format=LOG_FORMAT,
level=getattr(logging, opt.log_level.upper()))
logging.info(opt)
if int(opt.GPU) >= 0:
torch.cuda.set_device(int(opt.GPU))
if opt.load_checkpoint is not None:
logging.info("loading checkpoint from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2tree = checkpoint.model
input_vocab = checkpoint.input_vocab
else:
# Prepare dataset
src = SourceField()
nt = NTField()
pos = PosField()
tgt_tree = TreeField()
comp = CompField()
max_len = opt.max_len
def len_filter(example):
return len(example.src) <= max_len
train = torchtext.data.TabularDataset(path=opt.train_path,
format='tsv',
fields=[('src', src), ('nt', nt),
('pos', pos),
('tree', tgt_tree)],
filter_pred=len_filter)
dev = torchtext.data.TabularDataset(path=opt.dev_path,
format='tsv',
fields=[('src', src), ('nt', nt),
('pos', pos),
('tree', tgt_tree)],
filter_pred=len_filter)
src.build_vocab(train, max_size=50000)
comp.build_vocab(train, max_size=50000)
nt.build_vocab(train, max_size=50000)
pos.build_vocab(train, max_size=50000)
# src_tree.build_vocab(train, max_size=50000)
pos_in_nt = set()
for Pos in pos.vocab.stoi:
if nt.vocab.stoi[Pos] > 1:
pos_in_nt.add(nt.vocab.stoi[Pos])
hidden_size = opt.hidden_size
input_vocab = src.vocab
nt_vocab = nt.vocab
def tree_to_id(tree):
tree.set_label(nt_vocab.stoi[tree.label()])
if len(tree) == 1 and str(tree[0])[0] is not '(':
tree[0] = input_vocab.stoi[tree[0]]
return
else:
for subtree in tree:
tree_to_id(subtree)
tree.append(Tree(nt_vocab.stoi['<eos>'], []))
return tree
# train.examples = [str(tree_to_id(ex.tree)) for ex in train.examples]
# dev.examples = [str(tree_to_id(ex.tree)) for ex in dev.examples]
for ex in train.examples:
ex.tree = str(tree_to_id(Tree.fromstring(ex.tree)))
for ex in dev.examples:
ex.tree = str(tree_to_id(Tree.fromstring(ex.tree)))
# train.examples = [tree_to_id(Tree.fromstring(ex.tree)) for ex in train.examples]
# dev.examples = [str(tree_to_id(Tree.fromstring(ex.tree))) for ex in dev.examples]
if opt.word_embedding is not None:
input_vocab.load_vectors([opt.word_embedding])
loss = NLLLoss()
if torch.cuda.is_available():
loss.cuda()
loss.reset()
seq2tree = None
optimizer = None
if not opt.resume:
# Initialize model
bidirectional = opt.bidirectional_encoder
encoder = EncoderRNN(len(src.vocab),
opt.word_embedding_size,
max_len,
hidden_size,
bidirectional=bidirectional,
variable_lengths=True)
decoder = DecoderTree(len(src.vocab),
opt.word_embedding_size,
opt.nt_embedding_size,
len(nt.vocab),
max_len,
hidden_size *
2 if bidirectional else hidden_size,
sos_id=nt_vocab.stoi['<sos>'],
eos_id=nt_vocab.stoi['<eos>'],
dropout_p=0.2,
use_attention=True,
bidirectional=bidirectional,
pos_in_nt=pos_in_nt)
seq2tree = Seq2tree(encoder, decoder)
if torch.cuda.is_available():
seq2tree.cuda()
for param in seq2tree.parameters():
param.data.uniform_(-0.08, 0.08)
# encoder.embedding.weight.data.set_(input_vocab.vectors)
# encoder.embedding.weight.data.set_(output_vocab.vectors)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
optimizer = Optimizer(optim.Adam(seq2tree.parameters(), lr=opt.lr),
max_grad_norm=5)
# train
t = SupervisedTrainer(loss=loss,
batch_size=opt.batch_size,
checkpoint_every=opt.checkpoint_every,
print_every=10,
expt_dir=opt.expt_dir,
lr=opt.lr)
seq2tree = t.train(seq2tree,
train,
num_epochs=opt.epoch,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0,
resume=opt.resume)
predictor = Predictor(seq2tree, input_vocab, nt_vocab)
return predictor, dev, train
# TODO add dev eval here for early stopping
if config['train model']:
seq2seq = t.train(input_vocab,
feats_vocab,
seq2seq,
train,
num_epochs=config['epochs'],
vectors=vectors,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
# pdb.set_trace()
predictor = Predictor(seq2seq, input_vocab, feats_vocab, output_vocab, vectors)
# seq2top = Seq2seq(seq2seq.encoder, )
# topk_predictor = Predictor(seq2top, input_vocab, output_vocab, vectors)
if config['pull embeddings']:
out_vecs = {}
if config['feat embeddings']:
feats = {}
of = open(config['feat output'], 'wb')
# TODO add option to save output
src = SourceField()
feat = SourceField()
tgt = TargetField()
# pdb.set_trace()
# train
t = SupervisedTrainer(
loss=loss,
batch_size=32,
checkpoint_every=50,
print_every=10,
expt_dir=opt.expt_dir,
)
seq2seq = t.train(
seq2seq,
train,
num_epochs=6,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume,
)
evaluator = Evaluator(loss=loss, batch_size=32)
dev_loss, accuracy = evaluator.evaluate(seq2seq, dev)
assert dev_loss < 1.5
beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 3))
predictor = Predictor(beam_search, input_vocab, output_vocab)
inp_seq = "1 3 5 7 9"
seq = predictor.predict(inp_seq.split())
assert " ".join(seq[:-1]) == inp_seq[::-1]
def run_training(opt, default_data_dir, num_epochs=100):
if opt.load_checkpoint is not None:
logging.info("loading checkpoint from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME, opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
else:
# Prepare dataset
src = SourceField()
tgt = TargetField()
max_len = 50
data_file = os.path.join(default_data_dir, opt.train_path, 'data.txt')
logging.info("Starting new Training session on %s", data_file)
def len_filter(example):
return (len(example.src) <= max_len) and (len(example.tgt) <= max_len) \
and (len(example.src) > 0) and (len(example.tgt) > 0)
train = torchtext.data.TabularDataset(
path=data_file, format='json',
fields={'src': ('src', src), 'tgt': ('tgt', tgt)},
filter_pred=len_filter
)
dev = None
if opt.no_dev is False:
dev_data_file = os.path.join(default_data_dir, opt.train_path, 'dev-data.txt')
dev = torchtext.data.TabularDataset(
path=dev_data_file, format='json',
fields={'src': ('src', src), 'tgt': ('tgt', tgt)},
filter_pred=len_filter
)
src.build_vocab(train, max_size=50000)
tgt.build_vocab(train, max_size=50000)
input_vocab = src.vocab
output_vocab = tgt.vocab
# NOTE: If the source field name and the target field name
# are different from 'src' and 'tgt' respectively, they have
# to be set explicitly before any training or inference
# seq2seq.src_field_name = 'src'
# seq2seq.tgt_field_name = 'tgt'
# Prepare loss
weight = torch.ones(len(tgt.vocab))
pad = tgt.vocab.stoi[tgt.pad_token]
loss = Perplexity(weight, pad)
if torch.cuda.is_available():
logging.info("Yayyy We got CUDA!!!")
loss.cuda()
else:
logging.info("No cuda available device found running on cpu")
seq2seq = None
optimizer = None
if not opt.resume:
hidden_size = 128
decoder_hidden_size = hidden_size * 2
logging.info("EncoderRNN Hidden Size: %s", hidden_size)
logging.info("DecoderRNN Hidden Size: %s", decoder_hidden_size)
bidirectional = True
encoder = EncoderRNN(len(src.vocab), max_len, hidden_size,
bidirectional=bidirectional,
rnn_cell='lstm',
variable_lengths=True)
decoder = DecoderRNN(len(tgt.vocab), max_len, decoder_hidden_size,
dropout_p=0, use_attention=True,
bidirectional=bidirectional,
rnn_cell='lstm',
eos_id=tgt.eos_id, sos_id=tgt.sos_id)
seq2seq = Seq2seq(encoder, decoder)
if torch.cuda.is_available():
seq2seq.cuda()
for param in seq2seq.parameters():
param.data.uniform_(-0.08, 0.08)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
scheduler = StepLR(optimizer.optimizer, 1)
optimizer.set_scheduler(scheduler)
# train
num_epochs = num_epochs
batch_size = 32
checkpoint_every = num_epochs / 10
print_every = num_epochs / 100
properties = dict(batch_size=batch_size,
checkpoint_every=checkpoint_every,
print_every=print_every, expt_dir=opt.expt_dir,
num_epochs=num_epochs,
teacher_forcing_ratio=0.5,
resume=opt.resume)
logging.info("Starting training with the following Properties %s", json.dumps(properties, indent=2))
t = SupervisedTrainer(loss=loss, batch_size=num_epochs,
checkpoint_every=checkpoint_every,
print_every=print_every, expt_dir=opt.expt_dir)
seq2seq = t.train(seq2seq, train,
num_epochs=num_epochs, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
evaluator = Evaluator(loss=loss, batch_size=batch_size)
if opt.no_dev is False:
dev_loss, accuracy = evaluator.evaluate(seq2seq, dev)
logging.info("Dev Loss: %s", dev_loss)
logging.info("Accuracy: %s", dev_loss)
beam_search = Seq2seq(seq2seq.encoder, TopKDecoder(seq2seq.decoder, 4))
predictor = Predictor(beam_search, input_vocab, output_vocab)
while True:
try:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
results = predictor.predict_n(seq, n=3)
for i, res in enumerate(results):
print('option %s: %s\n', i + 1, res)
except KeyboardInterrupt:
logging.info("Bye Bye")
exit(0)
max_grad_norm=args.max_grad_norm)
############### train model ################
t = SpkTrainer(args=args,
loss=loss,
batch_size=args.batch_size,
checkpoint_every=args.ckpt_every,
random_seed=args.seed,
print_every=args.verbose,
expt_dir=args.expt_dir)
discrim = t.train(model=model,
data=train,
num_epochs=args.epochs,
dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=args.resume)
################ initialize predictor ###################
predictor = Predictor(model=model,
src_vocab=input_vocab,
tgt_vocab=output_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
import requests
import json
from flask import Flask, request, Response
app = Flask(__name__, static_url_path='')
from seq2seq.evaluator import Predictor
from seq2seq.util.checkpoint import Checkpoint
cp = Checkpoint.load("./experiment/checkpoints/2019_08_02_13_23_47/")
predictor = Predictor(cp.model, cp.input_vocab, cp.output_vocab)
def q(s):
l = -len('<eos>')
return "".join(predictor.predict(list(s)))[:l]
@app.route("/api")
def get():
r = request.args.get('text', '')
if r == "":
return Response(json.dumps({'status':"error", 'message':"empty input"}))
return Response(json.dumps({'status':"ok", 'message':q(r)}),
mimetype="application/json")
@app.route('/')
def index():
return app.send_static_file('index.html')#
if __name__ == "__main__":
app.run(host='127.0.0.1', port=5002, debug=True)
parser.add_argument('--log-level',
dest='log_level',
default='info',
help='Logging level.')
opt = parser.parse_args()
LOG_FORMAT = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s'
logging.basicConfig(format=LOG_FORMAT,
level=getattr(logging, opt.log_level.upper()))
logging.info(opt)
if opt.load_checkpoint is not None:
logging.info("loading checkpoint from {}".format(
os.path.join(opt.expt_dir, Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)))
checkpoint_path = os.path.join(opt.expt_dir,
Checkpoint.CHECKPOINT_DIR_NAME,
opt.load_checkpoint)
checkpoint = Checkpoint.load(checkpoint_path)
seq2seq = checkpoint.model
input_vocab = checkpoint.input_vocab
output_vocab = checkpoint.output_vocab
predictor = Predictor(seq2seq, input_vocab, output_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
for param in seq2tree.parameters():
param.data.uniform_(-0.08, 0.08)
# encoder.embedding.weight.data.set_(input_vocab.vectors)
# encoder.embedding.weight.data.set_(output_vocab.vectors)
# Optimizer and learning rate scheduler can be customized by
# explicitly constructing the objects and pass to the trainer.
#
# optimizer = Optimizer(torch.optim.Adam(seq2seq.parameters()), max_grad_norm=5)
# scheduler = StepLR(optimizer.optimizer, 1)
# optimizer.set_scheduler(scheduler)
optimizer = Optimizer(optim.Adam(seq2tree.parameters(), lr=1e-4), max_grad_norm=5)
# train
t = SupervisedTrainer(loss=loss, batch_size=1,
checkpoint_every=50,
print_every=10, expt_dir=opt.expt_dir)
seq2tree = t.train(seq2tree, train,
num_epochs=20, dev_data=dev,
optimizer=optimizer,
teacher_forcing_ratio=0.5,
resume=opt.resume)
predictor = Predictor(seq2tree, input_vocab, input_vocab)
while True:
seq_str = raw_input("Type in a source sequence:")
seq = seq_str.strip().split()
print(predictor.predict(seq))
def __init__(
self,
task_path,
figsize=(15, 13),
decimals=2,
is_show_attn_split=True,
is_show_evaluation=True,
output_length_key='length',
attention_key="attention_score",
position_attn_key='position_attention',
content_attn_key='content_attention',
positional_table_labels={
"λ%": "position_percentage",
"C.γ": "content_confidence",
#"lgt": "approx_max_logit",
"C.λ": "pos_confidence",
"μ": "mu",
"σ": "sigma",
"w_α": "mean_attn_old_weight",
"w_j/n": "rel_counter_decoder_weight",
"w_1/n": "single_step_weight",
"w_μ": "mu_old_weight",
"w_γ": "mean_content_old_weight",
"w_1": "bias_weight"
},
# "% carry": "carry_rates",
is_show_name=True,
max_src=17,
max_out=13,
max_tgt=13,
**kwargs):
check = Checkpoint.load(task_path)
self.model = check.model
# store some interesting variables
self.model.set_dev_mode()
self.predictor = Predictor(self.model, check.input_vocab,
check.output_vocab)
self.model_name = task_path.split("/")[-2]
self.figsize = figsize
self.decimals = decimals
self.is_show_attn_split = is_show_attn_split
self.is_show_evaluation = is_show_evaluation
self.positional_table_labels = positional_table_labels
self.is_show_name = is_show_name
self.max_src = max_src
self.max_out = max_out
self.max_tgt = max_tgt
self.output_length_key = output_length_key
self.attention_key = attention_key
self.position_attn_key = position_attn_key
self.content_attn_key = content_attn_key
if self.is_show_evaluation:
self.is_symbol_rewriting = "symbol rewriting" in task_path.lower()
self.metric_computer = MetricComputer(
check, is_symbol_rewriting=self.is_symbol_rewriting, **kwargs)
if self.model.decoder.is_attention is None:
raise AttentionException("Model is not using attention.")
def eval_fa_equiv(model, data, input_vocab, output_vocab):
loss = NLLLoss()
batch_size = 1
model.eval()
loss.reset()
match = 0
total = 0
device = None if torch.cuda.is_available() else -1
batch_iterator = torchtext.data.BucketIterator(
dataset=data,
batch_size=batch_size,
sort=False,
sort_key=lambda x: len(x.src),
device=device,
train=False)
tgt_vocab = data.fields[seq2seq.tgt_field_name].vocab
pad = tgt_vocab.stoi[data.fields[seq2seq.tgt_field_name].pad_token]
predictor = Predictor(model, input_vocab, output_vocab)
num_samples = 0
perfect_samples = 0
dfa_perfect_samples = 0
match = 0
total = 0
with torch.no_grad():
for batch in batch_iterator:
num_samples = num_samples + 1
input_variables, input_lengths = getattr(batch,
seq2seq.src_field_name)
target_variables = getattr(batch, seq2seq.tgt_field_name)
target_string = decode_tensor(target_variables, output_vocab)
#target_string = target_string + " <eos>"
input_string = decode_tensor(input_variables, input_vocab)
generated_string = ' '.join([
x for x in predictor.predict(input_string.strip().split())[:-1]
if x != '<pad>'
])
#str(pos_example)[2]
generated_string = refine_outout(generated_string)
#str(pos_example)[2]
pos_example = subprocess.check_output([
'python2', 'regexDFAEquals.py', '--gold',
'{}'.format(target_string), '--predicted',
'{}'.format(generated_string)
])
if target_string == generated_string:
perfect_samples = perfect_samples + 1
dfa_perfect_samples = dfa_perfect_samples + 1
elif str(pos_example)[2] == '1':
dfa_perfect_samples = dfa_perfect_samples + 1
target_tokens = target_string.split()
generated_tokens = generated_string.split()
shorter_len = min(len(target_tokens), len(generated_tokens))
for idx in range(len(generated_tokens)):
total = total + 1
if idx >= len(target_tokens):
total = total + 1
elif target_tokens[idx] == generated_tokens[idx]:
match = match + 1
if total == 0:
accuracy = float('nan')
else:
accuracy = match / total
string_accuracy = perfect_samples / num_samples
dfa_accuracy = dfa_perfect_samples / num_samples
f = open('./time_logs/log_score_time.txt', 'a')
f.write('{}\n'.format(dfa_accuracy))
f.close()