def main(training_dir, dataset, test_dataset, config, test_config, model_name, model_type='gridTD',
dataset_type='validation',
beam_size=3,
max_caption_length=20):
# print(dataset.DATASET_NAME)
if dataset_type not in ['validation', 'test']:
raise ValueError('dataset_type={} is not recognized!'.format(
dataset_type))
model_path = os.path.join(training_dir, model_name)
logging('Loading model..')
dataset_provider = DatasetPreprocessorAttention(dataset, config, single_caption=True)
if 'gridTD' == model_type:
model = ImgCaptioninggridTDAdaptiveModel(config)
elif 'adaptive' == model_type:
model = ImgCaptioningAdaptiveAttentionModel(config)
else:
raise NotImplementedError('Please specify model_type as gridTD or adaptive')
model.build(dataset_provider.vocabs, dataset_provider.vocab_size)
keras_model = model.keras_model
logging('Loading model weights..')
keras_model.load_weights(model_path, by_name=True)
inference = BeamSearchInference(keras_model, dataset=dataset, config=config,
test_dataset=test_dataset, test_config=test_config,
beam_size=beam_size,
max_caption_length=max_caption_length)
logging('Evaluating {} set..'.format(dataset_type))
if dataset_type == 'test':
metrics, save_predictions, save_reference = inference.evaluate_test_set(
include_prediction=True)
elif dataset_type == 'train':
metrics, save_predictions, save_reference = inference.evaluate_training_set(
include_prediction=True)
else:
metrics, save_predictions, save_reference = inference.evaluate_validation_set(
include_prediction=True)
print(metrics)
logging('Writting result to files..')
metrics_path = os.path.join(training_dir,
'{}-metrics-{}-{}-{}-{}.yaml'.format(dataset_type, beam_size, model_name, test_dataset.DATASET_NAME,
max_caption_length))
save_predictions_path = os.path.join(training_dir,
'{}-predictions-{}-{}-{}-{}.yaml'.format(dataset_type, beam_size, model_name, test_dataset.DATASET_NAME,
max_caption_length))
save_reference_path = os.path.join(training_dir,
'{}-reference-{}-{}-{}-{}.yaml'.format(dataset_type, beam_size, model_name,
test_dataset.DATASET_NAME,
max_caption_length))
write_yaml_file(metrics, metrics_path)
write_yaml_file(save_predictions, save_predictions_path)
write_yaml_file(save_reference, save_reference_path)
logging('Done!')
def run(self):
io_utils.logging('Building model..')
self._model.build(self._dataset_provider.vocabs,
self._dataset_provider.vocab_size)
if self._model_weights_path:
io_utils.logging('Loading model weights from {}..'.format(
self._model_weights_path))
self.keras_model.load_weights(self._model_weights_path)
if self._stop_training:
self._stop_training = False
return
io_utils.logging('Training {} is starting..'.format(
self._training_label))
self.keras_model.fit_generator(
generator=self._dataset_provider.training_set(),
steps_per_epoch=self._dataset_provider.training_steps,
epochs=self._epochs,
validation_data=self._dataset_provider.test_set(),
validation_steps=self._dataset_provider.test_steps,
max_queue_size=self._max_q_size,
workers=self._workers,
callbacks=self._callbacks,
verbose=self._verbose)
self._stop_training = False
io_utils.logging('Training {} has finished.'.format(
self._training_label))
def run(self, save_idx, epoch_length):
# epoch_length = 20
io_utils.logging('Building model..')
self._model.build(self._dataset_provider.vocabs,
self._dataset_provider.vocab_size)
if self._model_weights_path:
io_utils.logging('Loading model weights from {}..'.format(
self._model_weights_path))
self.keras_model.load_weights(self._model_weights_path)
io_utils.logging('Training {} is starting..'.format(
self._training_label))
total_loss = np.zeros((epoch_length, 3))
dataset_loader = self._dataset_provider.training_set(
include_datum=False)
for i in range(save_idx * epoch_length):
previous_data = next(dataset_loader)
for epoch_num in range(self._epochs):
progbar = generic_utils.Progbar(epoch_length)
print('Epoch {}/{}'.format(epoch_num, self._epochs))
iter_num = 0
while True:
X, y = next(dataset_loader)
print(X[0].shape, y.shape)
y_pred = self.keras_model.predict_on_batch(
X + [np.zeros(y.shape)])[0]
lrp_model = LRPInferenceLayergridTD(
self.keras_model, self._dataset_provider,
self._model._hidden_dim, self._model._embedding_dim,
self.L, self.D, self._model.img_encoder, 'mean')
lrp_weight = lrp_model.call(X + [y_pred])
del lrp_model
gc.collect()
losses = self.keras_model.train_on_batch(
X + [lrp_weight], [y, y])
acc = losses[3]
total_loss[iter_num, 0] = losses[0]
total_loss[iter_num, 1] = losses[1]
total_loss[iter_num, 2] = acc
iter_num += 1
progbar.update(iter_num,
[('loss', np.mean(total_loss[:iter_num, 1])),
('acc', np.mean(total_loss[:iter_num, 2]))])
if iter_num == epoch_length:
self.keras_model.save_weights(
os.path.join(
self._result_dir,
'keras_model_{:02d}_{:.4f}.hdf5'.format(
save_idx, acc)))
print('loss: ',
np.mean(total_loss, axis=0)[0],
np.mean(total_loss, axis=0)[1])
print('acc: ', np.mean(total_loss, axis=0)[2])
break
io_utils.logging('Training {} has finished.'.format(
self._training_label))
def on_train_end(self, logs):
if self._stopped_epoch is not None and self._verbose > 0:
logging('Epoch {}: stop after {}'.format(self._stopped_epoch,
self._timedelta))
def on_train_end(self, logs):
if self._stopped_epoch is not None and self._verbose > 0:
logging('Stop because val loss explodes at epoch {}'.format(
self._stopped_epoch))