def setUp(self):
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
config = ModelConfig()
config.vocab_size = len(p.vocab_word)
config.char_vocab_size = len(p.vocab_char)
model = SeqLabeling(config, ntags=len(p.vocab_tag))
model.load(filepath=os.path.join(SAVE_ROOT, 'model_weights.h5'))
self.tagger = anago.Tagger(model, preprocessor=p)
self.sent = 'President Obama is speaking at the White House.'
def test_eval(self):
test_path = os.path.join(DATA_ROOT, 'test.txt')
x_test, y_test = load_data_and_labels(test_path)
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
config = ModelConfig()
config.vocab_size = len(p.vocab_word)
config.char_vocab_size = len(p.vocab_char)
model = SeqLabeling(config, ntags=len(p.vocab_tag))
model.load(filepath=os.path.join(SAVE_ROOT, 'model_weights.h5'))
evaluator = anago.Evaluator(model, preprocessor=p)
evaluator.eval(x_test, y_test)
def setUp(self):
SAVE_ROOT = os.path.join(os.path.dirname(__file__), '../models')
model_config = ModelConfig()
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
model_config.vocab_size = len(p.vocab_word)
model_config.char_vocab_size = len(p.vocab_char)
weights = 'model_weights.h5'
self.tagger = anago.Tagger(model_config,
weights,
save_path=SAVE_ROOT,
preprocessor=p)
self.sent = 'President Obama is speaking at the White House.'
def test_eval(self):
DATA_ROOT = os.path.join(os.path.dirname(__file__),
'../data/conll2003/en/tagging')
SAVE_ROOT = os.path.join(os.path.dirname(__file__), '../models')
model_config = ModelConfig()
test_path = os.path.join(DATA_ROOT, 'test.txt')
x_test, y_test = load_data_and_labels(test_path)
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
model_config.vocab_size = len(p.vocab_word)
model_config.char_vocab_size = len(p.vocab_char)
weights = 'model_weights.h5'
evaluator = anago.Evaluator(model_config,
weights,
save_path=SAVE_ROOT,
preprocessor=p)
evaluator.eval(x_test, y_test)
from anago.reader import load_word_embeddings, load_data_and_labels
DATA_ROOT = 'data/conll2003/en/ner'
LOAD_ROOT = './models' # trained model
LOG_ROOT = './logs' # checkpoint, tensorboard
embedding_path = '/media/jan/OS/Dataset/WordEmbeddings/wiki.en.vec'
model_config = ModelConfig()
test_path = os.path.join(DATA_ROOT, 'train.small.txt')
x_test, y_test = load_data_and_labels(test_path)
p = prepare_preprocessor(x_test, y_test)
embeddings = load_word_embeddings(p.vocab_word, embedding_path,
model_config.word_embedding_size)
model_config.vocab_size = len(p.vocab_word)
model_config.char_vocab_size = len(p.vocab_char)
model_path = os.path.join(LOAD_ROOT, 'mymodel.h5')
model = SeqLabeling(model_config, embeddings, len(p.vocab_tag))
model.load(model_path)
X, y = p.transform(x_test, y_test)
predictions = model.predict(X)
for words, prediction, sentence_length in zip(x_test, predictions, X[2]):
nopad_prediction = prediction[:sentence_length.item()]
label_indices = [np.argmax(x) for x in nopad_prediction]
labels = p.inverse_transform(label_indices)
print "\n".join(["{}\t{}".format(w, l) for w, l in zip(words, labels)])
print ''