def train(self, x_train, y_train, x_valid=None, y_valid=None):
# Prepare training and validation data(steps, generator)
train_steps, train_batches = batch_iter(
x_train,
y_train,
self.training_config.batch_size,
preprocessor=self.preprocessor)
valid_steps, valid_batches = batch_iter(
x_valid,
y_valid,
self.training_config.batch_size,
preprocessor=self.preprocessor)
self.model.compile(
loss=self.model.crf.loss,
optimizer=Adam(lr=self.training_config.learning_rate),
)
# Prepare callbacks
callbacks = get_callbacks(
log_dir=self.checkpoint_path,
tensorboard=self.tensorboard,
eary_stopping=self.training_config.early_stopping,
valid=(valid_steps, valid_batches, self.preprocessor))
# Train the model
self.model.fit_generator(generator=train_batches,
steps_per_epoch=train_steps,
epochs=self.training_config.max_epoch,
callbacks=callbacks)
def train(self, x_train, y_train, x_valid=None, y_valid=None):
# Prepare training and validation data(steps, generator)
train_steps, train_batches = batch_iter(
list(zip(x_train, y_train)),
self.training_config.batch_size,
preprocessor=self.preprocessor)
valid_steps, valid_batches = batch_iter(
list(zip(x_valid, y_valid)),
self.training_config.batch_size,
preprocessor=self.preprocessor)
# Build the model
model = SeqLabeling(self.model_config, self.embeddings,
len(self.preprocessor.vocab_tag))
model.compile(
loss=model.crf.loss,
optimizer=Adam(lr=self.training_config.learning_rate),
)
# Prepare callbacks for training
callbacks = get_callbacks(
log_dir=self.checkpoint_path,
tensorboard=self.tensorboard,
eary_stopping=self.training_config.early_stopping,
valid=(valid_steps, valid_batches, self.preprocessor))
# Train the model
model.fit_generator(generator=train_batches,
steps_per_epoch=train_steps,
epochs=self.training_config.max_epoch,
callbacks=callbacks)
# Save the model
model.save(os.path.join(self.save_path, 'model_weights.h5'))
def eval(self, x_test, y_test):
# Prepare test data(steps, generator)
train_steps, train_batches = batch_iter(
x_test,
y_test,
batch_size=20, # Todo: if batch_size=1, eval does not work.
shuffle=False,
preprocessor=self.preprocessor)
# Build the evaluator and evaluate the model
f1score = F1score(train_steps, train_batches, self.preprocessor)
f1score.model = self.model
f1score.on_epoch_end(epoch=-1) # epoch takes any integer.
def eval(self, x_test, y_test):
# Prepare test data(steps, generator)
train_steps, train_batches = batch_iter(list(zip(x_test, y_test)),
self.config.batch_size,
preprocessor=self.preprocessor)
# Build the model
model = SeqLabeling(self.config,
ntags=len(self.preprocessor.vocab_tag))
model.load(filepath=os.path.join(self.save_path, self.weights))
# Build the evaluator and evaluate the model
f1score = F1score(train_steps, train_batches, self.preprocessor)
f1score.model = model
f1score.on_epoch_end(epoch=-1) # epoch takes any integer.
def test_batch_iter(self):
sents, labels = load_data_and_labels(self.filename)
batch_size = 32
p = prepare_preprocessor(sents, labels)
steps, batches = batch_iter(list(zip(sents, labels)), batch_size, preprocessor=p)
self.assertEqual(len([_ for _ in batches]), steps) # Todo: infinite loop