def infer():
premises_txt = get_sentences(premises_path)
hypotheses_txt = get_sentences(hypotheses_path)
# both should be tuples. [0] is a torch.tensor of size (max_len x N x 300)
# [1] is a torch.tensor of size (N)
# get glove
print("GloVe:\t\tloading...", end='\r')
glove = torchtext.vocab.GloVe()
print("GloVe:\t\tloaded ")
# get the batches in the right, embedded format
p_batch = get_batch(premises_txt, glove)
h_batch = get_batch(hypotheses_txt, glove)
# remove glove from memory
glove = None
# check that there's an equal number of pairs
assert p_batch[1].size() == h_batch[1].size()
n_sentences = len(p_batch[1])
# load the right model
model_path = 'models/' + model_type + '_model.tar'
assert os.path.exists(model_path)
print("Using", model_type.upper(), "model")
if model_type == 'baseline':
encoder = Baseline().to(device)
elif model_type == 'lstm':
encoder = LSTM().to(device)
elif model_type == 'bilstm':
encoder = BiLSTM().to(device)
elif model_type == 'maxbilstm':
encoder = BiLSTM(maxpooling=True).to(device)
else:
encoder = Baseline().to(device)
classifier = MLPClassifier(encoder, n_sentences).to(device)
encoder, classifier = load_model(encoder, classifier, model_path)
# get predictions
y = classifier.forward(p_batch, h_batch)
predictions = [outcome[l] for l in y.argmax(1).cpu().numpy()]
# print result
for i in range(n_sentences):
print("\nPREMISE:\t", premises_txt[i])
print("HYPOTHESIS:\t", hypotheses_txt[i])
print("PREDICTION:\t", predictions[i])
def SequenceEncoder():
if seq_encoder == 'BiLSTM':
encoder = BiLSTM(**kwargs)
elif seq_encoder == 'multi_CNN':
encoder = multi_CNN(**kwargs)
else:
print("please choose seq_encoder in ['BiLSTM','multi_CNN']")
return encoder
def TextEncoder():
if text_encoder == 'BiLSTM':
encoder = BiLSTM(**kwargs)
elif text_encoder == 'multi_CNN':
encoder = multi_CNN(**kwargs)
elif text_encoder == 'Transformer':
encoder = Transformer(**kwargs)
else:
print(
"please choose text_encoder in ['BiLSTM','multi_CNN','Transformer']"
)
return encoder
for gpu in gpus:
tf.config.experimental.set_memory_growth(gpu, True)
logical_gpus = tf.config.experimental.list_logical_devices('GPU')
print(len(gpus), "Physical GPUs,", len(logical_gpus), "Logical GPUs")
except RuntimeError as e:
print(e)
dataset_val, tokenizer, data_size = make_val_dataset('val')
def seq_to_str(seq):
output = ' '.join(tokenizer.sequences_to_texts([np.array(seq)]))
return output.replace(config.start_token, ' ').replace(config.end_token, ' ').replace('<s>', ' ').replace('</s>', ' ')
encoder = BiLSTM(config.vocab_size, config.embedding_len, config.encoder_feature_len)
decoder_left = Decoder(config.vocab_size, config.embedding_len, config.decoder_hidden_len)
decoder_right = Decoder(config.vocab_size, config.embedding_len, config.decoder_hidden_len)
optimizer = tf.keras.optimizers.Adam()
checkpoint = tf.train.Checkpoint(optimizer=optimizer,
encoder=encoder,
decoder_left=decoder_left,
decoder_right=decoder_right)
checkpoint.restore(tf.train.latest_checkpoint(config.checkpoint_dir))
hypothesis = []
reference = []
pbar = tqdm(total=data_size)
for article, abstract, guiding_objects, object_sequence in dataset_val:
hypothesis.append(predict(article, guiding_objects, object_sequence, encoder, decoder_left, decoder_right, tokenizer, config.beam_top_k))
reference.append(abstract)
def test(model_folder='models/', data_path='.data/'):
train_set, dev_set, test_set = reduce_dataset(full_train_set,
full_dev_set,
full_test_set,
n_samples=data_limit)
for encoder_type in encoder_types:
print()
dev_accuracies = []
test_accuracies = []
model_path = model_folder + encoder_type + '_model.tar'
if not os.path.exists(model_path):
print(encoder_type.upper(), "not found")
continue
print("Retrieving", encoder_type.upper(), "model")
if encoder_type == 'baseline':
encoder = Baseline().to(device)
elif encoder_type == 'lstm':
encoder = LSTM().to(device)
elif encoder_type == 'bilstm':
encoder = BiLSTM().to(device)
elif encoder_type == 'maxbilstm':
encoder = BiLSTM(maxpooling=True).to(device)
else:
encoder = Baseline().to(device)
classifier = MLPClassifier(encoder, batch_size).to(device)
encoder, classifier, end_epoch = load_model(encoder, classifier,
model_path)
train_iter, dev_iter, test_iter = torchtext.data.BucketIterator.splits(
datasets=(train_set, dev_set, test_set),
batch_sizes=(batch_size, batch_size, batch_size),
device=device,
shuffle=True)
# iteration of dev
for batch in dev_iter:
# p_batch and h_batch are tuples. The first element is the
# embedded batch, and the second contains all sentence lengths
p_batch, h_batch, l_batch = preprocess_batch(batch)
# forward pass
preds = classifier.forward(p_batch, h_batch)
# compute accuracies
dev_accuracies.append(get_accuracy(preds, l_batch))
dev_accuracy = np.mean(dev_accuracies)
# iteration of test
for batch in test_iter:
# p_batch and h_batch are tuples. The first element is the
# embedded batch, and the second contains all sentence lengths
p_batch, h_batch, l_batch = preprocess_batch(batch)
# forward pass
preds = classifier.forward(p_batch, h_batch)
# compute accuracies
test_accuracies.append(get_accuracy(preds, l_batch))
test_accuracy = np.mean(test_accuracies)
print("Test accuracy: ", round(test_accuracy * 100, 1), "%")
print("Dev accuracy: ", round(dev_accuracy * 100, 1), "%")
print("Total training epochs:", end_epoch)
def stest():
for encoder_type in encoder_types:
model_path = model_folder + encoder_type + '_model.tar'
assert os.path.exists(model_path)
print("Using", encoder_type.upper(), "model")
if encoder_type == 'baseline':
encoder = Baseline().to(device)
elif encoder_type == 'lstm':
encoder = LSTM().to(device)
elif encoder_type == 'bilstm':
encoder = BiLSTM().to(device)
elif encoder_type == 'maxbilstm':
encoder = BiLSTM(maxpooling=True).to(device)
else:
encoder = Baseline().to(device)
encoder = load_encoder(encoder, model_path)
# set parameters for senteval
params_senteval = {
'task_path': 'SentEval/data',
'usepytorch': True,
'kfold': 5,
'classifier': {
'nhid': 0,
'optim': 'rmsprop',
'batch_size': 128,
'tenacity': 3,
'epoch_size': 2
},
'encoder': encoder.to(device)
}
# senteval engine
se = senteval.engine.SE(params_senteval, batcher, prepare)
# task list
if task_size == 'reduced':
transfer_tasks = ['MR', 'CR', 'SUBJ', 'TREC', 'STS14']
else:
transfer_tasks = [
'CR', 'MR', 'MPQA', 'SUBJ', 'SST2', 'SST5', 'TREC', 'MRPC',
'SNLI', 'SICKEntailment', 'SICKRelatedness', 'STSBenchmark',
'ImageCaptionRetrieval', 'STS12', 'STS13', 'STS14', 'STS15',
'STS16', 'Length', 'WordContent', 'Depth', 'TopConstituents',
'BigramShift', 'Tense', 'SubjNumber', 'ObjNumber', 'OddManOut',
'CoordinationInversion'
]
# run evaluation
results = se.eval(transfer_tasks)
# print
print(results)
# define save path
result_path = 'senteval_results/'
if not os.path.exists(result_path):
os.makedirs(result_path)
time_string = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
file_path = result_path + time_string + '_' + encoder_type
# save
with open(file_path, 'wb') as file:
pickle.dump(results, file)