def test_train(self):
model_config = ModelConfig()
training_config = TrainingConfig()
train_path = os.path.join(DATA_ROOT, 'train.txt')
valid_path = os.path.join(DATA_ROOT, 'valid.txt')
x_train, y_train = load_data_and_labels(train_path)
x_valid, y_valid = load_data_and_labels(valid_path)
p = prepare_preprocessor(x_train, y_train)
p.save(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
embeddings = load_word_embeddings(p.vocab_word, EMBEDDING_PATH,
model_config.word_embedding_size)
model_config.char_vocab_size = len(p.vocab_char)
model = SeqLabeling(model_config, embeddings, len(p.vocab_tag))
trainer = anago.Trainer(model,
training_config,
checkpoint_path=LOG_ROOT,
save_path=SAVE_ROOT,
preprocessor=p,
embeddings=embeddings)
trainer.train(x_train, y_train, x_valid, y_valid)
model.save(os.path.join(SAVE_ROOT, 'model_weights.h5'))
def __init__(self,
char_emb_size=25,
word_emb_size=100,
char_lstm_units=25,
word_lstm_units=100,
dropout=0.5,
char_feature=True,
crf=True,
batch_size=1024,
optimizer='adam',
learning_rate=0.001,
lr_decay=0.9,
clip_gradients=5.0,
max_epoch=15,
early_stopping=True,
patience=3,
train_embeddings=True,
max_checkpoints_to_keep=5,
log_dir=None,
embeddings=()):
self.model_config = ModelConfig(char_emb_size, word_emb_size,
char_lstm_units, word_lstm_units,
dropout, char_feature, crf)
self.training_config = TrainingConfig(batch_size, optimizer,
learning_rate, lr_decay,
clip_gradients, max_epoch,
early_stopping, patience,
train_embeddings,
max_checkpoints_to_keep)
self.model = None
self.p = None
self.log_dir = log_dir
self.embeddings = embeddings
def test_train(self):
DATA_ROOT = os.path.join(os.path.dirname(__file__),
'../data/conll2003/en/tagging')
SAVE_ROOT = os.path.join(os.path.dirname(__file__),
'../models') # trained model
LOG_ROOT = os.path.join(os.path.dirname(__file__),
'../logs') # checkpoint, tensorboard
embedding_path = os.path.join(os.path.dirname(__file__),
'../data/glove.6B/glove.6B.100d.txt')
model_config = ModelConfig()
training_config = TrainingConfig()
train_path = os.path.join(DATA_ROOT, 'train.txt')
valid_path = os.path.join(DATA_ROOT, 'valid.txt')
test_path = os.path.join(DATA_ROOT, 'test.txt')
x_train, y_train = load_data_and_labels(train_path)
x_valid, y_valid = load_data_and_labels(valid_path)
x_test, y_test = load_data_and_labels(test_path)
p = prepare_preprocessor(np.r_[x_train, x_valid, x_test],
y_train) # np.r_ is for vocabulary expansion.
p.save(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
embeddings = load_word_embeddings(p.vocab_word, embedding_path,
model_config.word_embedding_size)
model_config.char_vocab_size = len(p.vocab_char)
trainer = anago.Trainer(model_config,
training_config,
checkpoint_path=LOG_ROOT,
save_path=SAVE_ROOT,
preprocessor=p,
embeddings=embeddings)
trainer.train(x_train, y_train, x_test, y_test)
def setUp(self):
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
config = ModelConfig()
config.vocab_size = len(p.vocab_word)
config.char_vocab_size = len(p.vocab_char)
model = SeqLabeling(config, ntags=len(p.vocab_tag))
model.load(filepath=os.path.join(SAVE_ROOT, 'model_weights.h5'))
self.tagger = anago.Tagger(model, preprocessor=p)
self.sent = 'President Obama is speaking at the White House.'
def test_eval(self):
test_path = os.path.join(DATA_ROOT, 'test.txt')
x_test, y_test = load_data_and_labels(test_path)
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
config = ModelConfig()
config.vocab_size = len(p.vocab_word)
config.char_vocab_size = len(p.vocab_char)
model = SeqLabeling(config, ntags=len(p.vocab_tag))
model.load(filepath=os.path.join(SAVE_ROOT, 'model_weights.h5'))
evaluator = anago.Evaluator(model, preprocessor=p)
evaluator.eval(x_test, y_test)
def setUp(self):
SAVE_ROOT = os.path.join(os.path.dirname(__file__), '../models')
model_config = ModelConfig()
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
model_config.vocab_size = len(p.vocab_word)
model_config.char_vocab_size = len(p.vocab_char)
weights = 'model_weights.h5'
self.tagger = anago.Tagger(model_config,
weights,
save_path=SAVE_ROOT,
preprocessor=p)
self.sent = 'President Obama is speaking at the White House.'
def setUp(self):
self.model_config = ModelConfig()
self.training_config = TrainingConfig()
vocab = 10000
self.model_config.char_vocab_size = 80
self.embeddings = np.zeros((vocab, self.model_config.word_embedding_size))
self.filename = os.path.join(os.path.dirname(__file__), '../data/conll2003/en/ner/test.txt')
self.valid_file = os.path.join(os.path.dirname(__file__), '../data/conll2003/en/ner/valid.txt')
def test_eval(self):
DATA_ROOT = os.path.join(os.path.dirname(__file__),
'../data/conll2003/en/tagging')
SAVE_ROOT = os.path.join(os.path.dirname(__file__), '../models')
model_config = ModelConfig()
test_path = os.path.join(DATA_ROOT, 'test.txt')
x_test, y_test = load_data_and_labels(test_path)
p = WordPreprocessor.load(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
model_config.vocab_size = len(p.vocab_word)
model_config.char_vocab_size = len(p.vocab_char)
weights = 'model_weights.h5'
evaluator = anago.Evaluator(model_config,
weights,
save_path=SAVE_ROOT,
preprocessor=p)
evaluator.eval(x_test, y_test)
def load(cls, dir_path):
self = cls()
self.p = WordPreprocessor.load(
os.path.join(dir_path, cls.preprocessor_file))
config = ModelConfig.load(os.path.join(dir_path, cls.config_file))
dummy_embeddings = np.zeros(
(config.vocab_size, config.word_embedding_size), dtype=np.float32)
self.model = SeqLabeling(config,
dummy_embeddings,
ntags=len(self.p.vocab_tag))
self.model.load(filepath=os.path.join(dir_path, cls.weight_file))
return self
import os
import anago
from anago.preprocess import prepare_preprocessor
from anago.config import ModelConfig, TrainingConfig
from anago.models import SeqLabeling
import numpy as np
from anago.reader import load_word_embeddings, load_data_and_labels
DATA_ROOT = 'data/conll2003/en/ner'
LOAD_ROOT = './models' # trained model
LOG_ROOT = './logs' # checkpoint, tensorboard
embedding_path = '/media/jan/OS/Dataset/WordEmbeddings/wiki.en.vec'
model_config = ModelConfig()
test_path = os.path.join(DATA_ROOT, 'train.small.txt')
x_test, y_test = load_data_and_labels(test_path)
p = prepare_preprocessor(x_test, y_test)
embeddings = load_word_embeddings(p.vocab_word, embedding_path,
model_config.word_embedding_size)
model_config.vocab_size = len(p.vocab_word)
model_config.char_vocab_size = len(p.vocab_char)
model_path = os.path.join(LOAD_ROOT, 'mymodel.h5')
model = SeqLabeling(model_config, embeddings, len(p.vocab_tag))
model.load(model_path)
X, y = p.transform(x_test, y_test)
predictions = model.predict(X)
for words, prediction, sentence_length in zip(x_test, predictions, X[2]):
import os
from anago.config import ModelConfig
from anago.tagger import Tagger as tag
import anago
from CRF.anago.data import load_data_and_labels
from CRF.anago.data import prepare_preprocessor
DATA_ROOT = 'data/phenebank/'
train_path = os.path.join(DATA_ROOT, 'train.txt')
x_train, y_train = load_data_and_labels(train_path)
p = prepare_preprocessor(x_train, y_train)
model_config = ModelConfig()
SAVE_ROOT = './models' # trained model
weights = 'model_weights.h5'
tagger = anago.Tagger(model_config,
weights,
save_path=SAVE_ROOT,
preprocessor=p)
test_path = "data/phenebank/test.txt"
with open(test_path) as ifile:
this_sentence = []
all_sentences = []
this_output = []
all_outputs = []
for line in ifile:
class Sequence(object):
config_file = 'config.json'
weight_file = 'model_weights.h5'
preprocessor_file = 'preprocessor.pkl'
def __init__(self,
char_emb_size=25,
word_emb_size=100,
char_lstm_units=25,
word_lstm_units=100,
dropout=0.5,
char_feature=True,
crf=True,
batch_size=1024,
optimizer='adam',
learning_rate=0.001,
lr_decay=0.9,
clip_gradients=5.0,
max_epoch=15,
early_stopping=True,
patience=3,
train_embeddings=True,
max_checkpoints_to_keep=5,
log_dir=None,
embeddings=()):
self.model_config = ModelConfig(char_emb_size, word_emb_size,
char_lstm_units, word_lstm_units,
dropout, char_feature, crf)
self.training_config = TrainingConfig(batch_size, optimizer,
learning_rate, lr_decay,
clip_gradients, max_epoch,
early_stopping, patience,
train_embeddings,
max_checkpoints_to_keep)
self.model = None
self.p = None
self.log_dir = log_dir
self.embeddings = embeddings
def train(self,
x_train,
y_train,
x_valid=None,
y_valid=None,
vocab_init=None,
verbose=1):
self.p = prepare_preprocessor(x_train, y_train, vocab_init=vocab_init)
embeddings = filter_embeddings(self.embeddings, self.p.vocab_word,
self.model_config.word_embedding_size)
self.model_config.vocab_size = len(self.p.vocab_word)
self.model_config.char_vocab_size = len(self.p.vocab_char)
self.model = SeqLabeling(self.model_config, embeddings,
len(self.p.vocab_tag))
if not os.path.exists(self.log_dir):
print('Successfully made a directory: {}'.format(self.log_dir))
os.mkdir(self.log_dir)
self.p.save(os.path.join(self.log_dir, self.preprocessor_file))
self.model_config.save(os.path.join(self.log_dir, self.config_file))
print('Successfully save config and preprocess files')
trainer = Trainer(self.model,
self.training_config,
checkpoint_path=self.log_dir,
preprocessor=self.p)
return trainer.train(x_train, y_train, x_valid, y_valid, verbose)
def eval(self, x_test, y_test):
if self.model:
evaluator = Evaluator(self.model, preprocessor=self.p)
evaluator.eval(x_test, y_test)
else:
raise (OSError('Could not find a model. Call load(dir_path).'))
def analyze(self, words):
if self.model:
tagger = Tagger(self.model, preprocessor=self.p)
return tagger.analyze(words)
else:
raise (OSError('Could not find a model. Call load(dir_path).'))
def save(self, dir_path):
self.p.save(os.path.join(dir_path, self.preprocessor_file))
self.model_config.save(os.path.join(dir_path, self.config_file))
self.model.save(os.path.join(dir_path, self.weight_file))
@classmethod
def load(cls, dir_path):
self = cls()
self.p = WordPreprocessor.load(
os.path.join(dir_path, cls.preprocessor_file))
config = ModelConfig.load(os.path.join(dir_path, cls.config_file))
dummy_embeddings = np.zeros(
(config.vocab_size, config.word_embedding_size), dtype=np.float32)
self.model = SeqLabeling(config,
dummy_embeddings,
ntags=len(self.p.vocab_tag))
self.model.load(filepath=os.path.join(dir_path, cls.weight_file))
self.model._make_predict_function()
return self
