def train(self, x_train, y_train, x_valid=None, y_valid=None):
# Prepare training and validation data(steps, generator)
train_steps, train_batches = batch_iter(
x_train,
y_train,
self.training_config.batch_size,
preprocessor=self.preprocessor)
valid_steps, valid_batches = batch_iter(
x_valid,
y_valid,
self.training_config.batch_size,
preprocessor=self.preprocessor)
self.model.compile(
loss=self.model.crf.loss,
optimizer=Adam(lr=self.training_config.learning_rate),
)
# Prepare callbacks
callbacks = get_callbacks(
log_dir=self.checkpoint_path,
tensorboard=self.tensorboard,
eary_stopping=self.training_config.early_stopping,
valid=(valid_steps, valid_batches, self.preprocessor))
# Train the model
self.model.fit_generator(generator=train_batches,
steps_per_epoch=train_steps,
epochs=self.training_config.max_epoch,
callbacks=callbacks)
def train(self, X_train, Y_train, X_valid=None, Y_valid=None):
# Prepare training and validation data(steps, generator)
train_steps, train_batches = batch_iter(
list(zip(X_train, Y_train)),
self.training_config.batch_size,
preprocessor=self.preprocessor)
X_valid, Y_valid = self.preprocessor.transform(X_valid, Y_valid)
# Build the model
class_ = getattr(anago.models, self.keras_model_name)
model = class_(config=self.model_config,
w_embeddings=self.W_embeddings,
c_embeddings=self.C_embeddings,
pos_embeddings=self.POS_embeddings,
ntags=len(self.preprocessor.vocab_tag))
model.compile(loss=model.crf.loss,
optimizer=Adam(lr=self.training_config.learning_rate))
# Prepare callbacks for training
callbacks = get_mycallbacks(
save_path=os.path.join(self.save_path, self.model_name),
eary_stopping=self.training_config.early_stopping,
valid=(X_valid, Y_valid, self.preprocessor),
patience=self.training_config.patience)
# Train the model
model.fit_generator(generator=train_batches,
steps_per_epoch=train_steps,
epochs=self.training_config.max_epoch,
callbacks=callbacks)
def test_batch_iter(self):
sents, labels = load_data_and_labels(self.filename)
batch_size = 32
p = prepare_preprocessor(sents, labels)
steps, batches = batch_iter(list(zip(sents, labels)),
batch_size,
preprocessor=p)
self.assertEqual(len([_ for _ in batches]),
steps) # Todo: infinite loop
def eval(self, x_test, y_test):
# Prepare test data(steps, generator)
train_steps, train_batches = batch_iter(
x_test,
y_test,
batch_size=20, # Todo: if batch_size=1, eval does not work.
shuffle=False,
preprocessor=self.preprocessor)
# Build the evaluator and evaluate the model
f1score = F1score(train_steps, train_batches, self.preprocessor)
f1score.model = self.model
f1score.on_epoch_end(epoch=-1) # epoch takes any integer.