def test_predict():
global model, X_tr, Y_tr, word_to_ix, tag_to_ix
torch.manual_seed(765)
best_tags = model.predict(bilstm.prepare_sequence(X_tr[5], word_to_ix))
eq_(best_tags[0:5], ['SYM', 'NUM', 'INTJ', 'PART', 'SYM'])
best_tags = model.predict(bilstm.prepare_sequence(X_tr[0], word_to_ix))
eq_(best_tags[0:5], ['SYM', 'NUM', 'INTJ', 'PART', 'SYM'])
def test_forward_alg():
global model, X_tr, word_to_ix
torch.manual_seed(765)
lstm_feats = model.forward(bilstm.prepare_sequence(X_tr[0], word_to_ix))
alpha = model.forward_alg(lstm_feats)
assert_almost_equal(alpha.data.numpy()[0], 96.48747, places=4)
lstm_feats = model.forward(bilstm.prepare_sequence(X_tr[1], word_to_ix))
alpha = model.forward_alg(lstm_feats)
assert_almost_equal(alpha.data.numpy()[0], 59.80174, places=4)
def test_neg_log_likelihood():
global model, X_tr, Y_tr, word_to_ix, tag_to_ix
torch.manual_seed(765)
lstm_feats = model.forward(bilstm.prepare_sequence(X_tr[5], word_to_ix))
loss = model.neg_log_likelihood(
lstm_feats, bilstm.prepare_sequence(Y_tr[5], tag_to_ix))
assert_almost_equal(loss.data.numpy()[0], 45.898315, places=4)
lstm_feats = model.forward(bilstm.prepare_sequence(X_tr[0], word_to_ix))
loss = model.neg_log_likelihood(
lstm_feats, bilstm.prepare_sequence(Y_tr[0], tag_to_ix))
assert_almost_equal(loss.data.numpy()[0], 107.71689, places=4)
def test_score_sentence():
global model, X_tr, Y_tr, word_to_ix, tag_to_ix
torch.manual_seed(765)
lstm_feats = model.forward(bilstm.prepare_sequence(X_tr[0], word_to_ix))
score = model.score_sentence(lstm_feats,
bilstm.prepare_sequence(Y_tr[0], tag_to_ix))
assert_almost_equal(score.data.numpy()[0], -11.368162, places=4)
lstm_feats = model.forward(bilstm.prepare_sequence(X_tr[1], word_to_ix))
score = model.score_sentence(lstm_feats,
bilstm.prepare_sequence(Y_tr[1], tag_to_ix))
assert_almost_equal(score.data.numpy()[0], -3.9872737, places=4)
def apply_model(model,
outfilename,
word_to_ix,
all_tags=None,
trainfile=TRAIN_FILE,
testfile=DEV_FILE):
"""
applies the model on the data and writes the best sequence of tags to the outfile
"""
if all_tags is None:
all_tags = set()
# this is slow
for i, (words,
tags) in enumerate(preproc.conll_seq_generator(trainfile)):
for tag in tags:
all_tags.add(tag)
with open(outfilename, 'w') as outfile:
for words, _ in preproc.conll_seq_generator(testfile):
seq_words = bilstm.prepare_sequence(words, word_to_ix)
pred_tags = model.predict(seq_words)
for i, tag in enumerate(pred_tags):
outfile.write(tag + '\n')
outfile.write('\n')
def test_dlmodel_forward():
global model, X_tr, word_to_ix
torch.manual_seed(765);
lstm_feats = model(bilstm.prepare_sequence(X_tr[0], word_to_ix))
assert_almost_equal(lstm_feats[0].data.numpy()[0],-0.12883559, places=4)
assert_almost_equal(lstm_feats[0].data.numpy()[1],-0.42470014, places=4)
assert_almost_equal(lstm_feats[0].data.numpy()[2],-0.31790152, places=4)
def test_dlmodel_forward():
global model, X_tr, word_to_ix
torch.manual_seed(765)
lstm_feats = model(bilstm.prepare_sequence(X_tr[0], word_to_ix))
assert_almost_equal(lstm_feats[0].data.numpy()[0], -0.12883559, places=4)
assert_almost_equal(lstm_feats[0].data.numpy()[1], -0.42470014, places=4)
assert_almost_equal(lstm_feats[0].data.numpy()[2], -0.31790152, places=4)
def apply_model(model,outfilename,word_to_ix, all_tags=None,trainfile=TRAIN_FILE,testfile=DEV_FILE):
"""
applies the model on the data and writes the best sequence of tags to the outfile
"""
if all_tags is None:
all_tags = set()
# this is slow
for i,(words, tags) in enumerate(preproc.conll_seq_generator(trainfile)):
for tag in tags:
all_tags.add(tag)
with open(outfilename,'w') as outfile:
for words,_ in preproc.conll_seq_generator(testfile):
seq_words = bilstm.prepare_sequence(words, word_to_ix)
pred_tags = model.predict(seq_words)
for i,tag in enumerate(pred_tags):
outfile.write(tag+'\n')
outfile.write('\n')
