def train(self):
train_dir, summary_writer = setup_train_dir(self.config)
optimizer, params = get_optimizer(self.model, self.config)
exp_loss, best_dev_f1 = None, None
batch_iterator = DatasetConll2003(DatasetConll2003.DATA_TYPE_TRAIN,
self.config,
self.vocab,
is_train=True)
logging.info("Beginning training loop...")
epoch_tic = time.time()
pre_epoch = 0
for batch in batch_iterator:
epoch, global_step = batch_iterator.epoch, batch_iterator.iterations
iter_tic = time.time()
train_loss, param_norm, grad_norm = self.train_one_batch(
batch, optimizer, params)
write_summary(train_loss, "train/loss", summary_writer,
global_step)
iter_toc = time.time()
iter_time = iter_toc - iter_tic
exp_loss = 0.99 * exp_loss + 0.01 * train_loss if exp_loss else train_loss
if global_step % self.config.print_every == 0:
logging.info(
'epoch %d, iter %d, loss %.5f, smoothed loss %.5f, grad norm %.5f, param norm %.5f, batch time %.3f'
% (epoch, global_step, train_loss, exp_loss, grad_norm,
param_norm, iter_time))
if pre_epoch < epoch:
epoch_toc = time.time()
logging.info("End of epoch %i. Time for epoch: %f" %
(epoch, epoch_toc - epoch_tic))
dev_f1 = self.evalute_test_dev(summary_writer, epoch,
global_step, exp_loss)
if best_dev_f1 is None or dev_f1 > best_dev_f1:
if best_dev_f1:
bestmodel_dir = os.path.join(train_dir, 'bestmodel')
logging.info("Saving to %s..." % bestmodel_dir)
save_model(self.model, optimizer, exp_loss,
global_step, epoch, bestmodel_dir)
best_dev_f1 = dev_f1
sys.stdout.flush()
pre_epoch = epoch
lr = self.config.lr / (1 + self.config.lr_decay * epoch)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if epoch >= self.config.num_epochs:
break
def build_ensemble(models: List[tf.keras.Model],
config: ml_collections.ConfigDict) -> tf.keras.Model:
"""Compiles and returns an averaged ensemble model."""
model = _build_ensemble_graph(models, config)
losses = {}
loss_weights = {}
for outcome in config.outcomes:
losses[outcome.name] = metrics.get_loss(outcome)
loss_weights[outcome.name] = outcome.get('loss_weight', 1.0)
model.compile(
optimizer=model_utils.get_optimizer(config),
loss=losses,
loss_weights=loss_weights,
metrics=metrics.get_metrics(config.outcomes))
return model
dense = BatchNormalization()(dense)
dense = Dropout(config_model_param["dropout_rate"])(dense)
dense = Dense(config_model_param["layers_size"][i + 1],
activation=config_model_param['hidden_activation'],
name="MLP_combine_" + str(i + 1))(dense)
dense = BatchNormalization()(dense)
dense = Dropout(config_model_param["dropout_rate"])(dense)
if config_model_param["predict_labels"]:
out_size = get_input_label_size(config_data)
else:
out_size = 1
out_labels = Dense(out_size,
activation=config_model_param['output_activation'],
name="MLP_out")(dense)
model = Model(inputs=[query, doc], outputs=out_labels)
optimizer = get_optimizer(config_model_param["optimizer"])(
lr=config_model_param["learning_rate"])
# https://www.pyimagesearch.com/2017/10/30/how-to-multi-gpu-training-with-keras-python-and-deep-learning/
if G <= 1:
print("[INFO] training with 0/1 GPU...")
model_gpu = model
else:
print("[INFO] training with {} GPUs...".format(G))
# we'll store a copy of the model on *every* GPU and then combine
# the results from the gradient updates on the CPU
model_gpu = make_parallel(model, G)
# make the model parallel
# model_gpu = multi_gpu_model(model_gpu, gpus=G)
learning_rate = FLAGS.learning_rate
if FLAGS.load_model_path != '' and FLAGS.load_model_path is not None:
visual_model = load_model(FLAGS.load_model_path)
if FLAGS.show_model_summary:
visual_model.summary()
training_stats_file = os.path.join(FLAGS.save_model_path,
".training_stats.json")
if os.path.isfile(training_stats_file):
training_stats = json.load(open(training_stats_file))
learning_rate = training_stats['lr']
print("Will continue from learning rate: {}".format(learning_rate))
else:
visual_model = model_factory.get_model(FLAGS)
opt = get_optimizer(FLAGS.optimizer_type, learning_rate)
if FLAGS.multi_label_classification:
visual_model.compile(
loss='binary_crossentropy',
optimizer=opt,
metrics=[metrics.BinaryAccuracy(threshold=FLAGS.multilabel_threshold)])
else:
visual_model.compile(loss='sparse_categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
training_stats_file = {}
try:
os.makedirs(FLAGS.save_model_path)
except: