def test_load(self):
model = CharNER(self.model_config, vocab_size=100, ntags=10)
path = 'data/test.h5'
model.save(path)
self.assertTrue(os.path.exists(path))
if os.path.exists(path):
os.remove(path)
def test_predict(self):
X, y = load_data_and_labels(self.filename)
p = prepare_preprocessor(X, y)
self.model_config.vocab_size = len(p.vocab_char)
model = CharNER(self.model_config, p.vocab_size(), p.tag_size())
model.predict(p.transform(X))
def test_train(self):
model_config = ModelConfig()
training_config = TrainingConfig()
# train_path = os.path.join(DATA_ROOT, 'conll.txt')
# valid_path = os.path.join(DATA_ROOT, 'conll.txt')
# x_train, y_train = load_data_and_labels(train_path)
# x_valid, y_valid = load_data_and_labels(valid_path)
path = os.path.join(DATA_ROOT, 'dataset.tsv')
X, y = load_data_and_labels(path)
from sklearn.model_selection import train_test_split
x_train, x_valid, y_train, y_valid = train_test_split(X,
y,
test_size=0.3,
random_state=42)
# p = prepare_preprocessor(x_train, y_train)
p = prepare_preprocessor(X, y)
p.save(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
model_file = os.path.join(SAVE_ROOT, 'model.h5')
model = CharNER(model_config, p.vocab_size(), p.tag_size())
trainer = namaco.Trainer(model,
model.loss,
training_config,
log_dir=LOG_ROOT,
save_path=model_file,
preprocessor=p)
trainer.train(x_train, y_train, x_valid, y_valid)
def test_train(self):
model_config = ModelConfig()
training_config = TrainingConfig()
train_path = os.path.join(DATA_ROOT, 'conll.txt')
valid_path = os.path.join(DATA_ROOT, 'conll.txt')
x_train, y_train = load_data_and_labels(train_path)
x_valid, y_valid = load_data_and_labels(valid_path)
p = prepare_preprocessor(x_train, y_train)
p.save(os.path.join(SAVE_ROOT, 'preprocessor.pkl'))
model_file = os.path.join(SAVE_ROOT, 'model.h5')
model = CharNER(model_config, p.vocab_size(), p.tag_size())
trainer = namaco.Trainer(model,
model.loss,
training_config,
log_dir=LOG_ROOT,
save_path=model_file,
preprocessor=p)
trainer.train(x_train, y_train, x_valid, y_valid)
def test_compile(self):
model = CharNER(self.model_config, vocab_size=100, ntags=10)
model.compile(loss=model.loss,
optimizer=Adam(lr=self.training_config.learning_rate))
def test_build(self):
model = CharNER(self.model_config, vocab_size=100, ntags=10)