def average_models(models, output_model, weights=None):
from keras_wrapper.cnn_model import loadModel, saveModel
if not isinstance(models, list):
raise AssertionError('You must give a list of models to average.')
if len(models) == 0:
raise AssertionError(
'You provided an empty list of models to average!')
model_weights = np.asarray(
[1. / len(models)] *
len(models), dtype=np.float32) if (weights is None) or (
weights == []) else np.asarray(weights, dtype=np.float32)
if len(model_weights) != len(models):
raise AssertionError(
'You must give a list of weights of the same size than the list of models.'
)
loaded_models = [loadModel(m, -1, full_path=True) for m in models]
# Check that all models are compatible
if not all(
[hasattr(loaded_model, 'model') for loaded_model in loaded_models]):
raise AssertionError('Not all models have the attribute "model".')
if not (all([
hasattr(loaded_model, 'model_init')
for loaded_model in loaded_models
]) or all([
not hasattr(loaded_model, 'model_init')
for loaded_model in loaded_models
])):
raise AssertionError('Not all models have the attribute "model_init".')
if not (all([
hasattr(loaded_model, 'model_next')
for loaded_model in loaded_models
]) or all([
not hasattr(loaded_model, 'model_next')
for loaded_model in loaded_models
])):
raise AssertionError('Not all models have the attribute "model_next".')
# Check all layers are the same
if not (all([[
str(loaded_models[0].model.weights[i]) == str(
loaded_model.model.weights[i])
for i in range(len(loaded_models[0].model.weights))
] for loaded_model in loaded_models])):
raise AssertionError('Not all models have the same weights!')
if hasattr(loaded_models[0], 'model_init'):
if not all([[
str(loaded_models[0].model_init.weights[i]) == str(
loaded_model.model_init.weights[i])
for i in range(len(loaded_models[0].model.weights))
] for loaded_model in loaded_models]):
raise AssertionError('Not all models have the same weights!')
if not all([[
str(loaded_models[0].model.weights[i]) == str(
loaded_model.model.weights[i])
for i in range(len(loaded_models[0].model_init.weights))
] for loaded_model in loaded_models]):
raise AssertionError('Not all model_inits have the same weights!')
if hasattr(loaded_models[0], 'model_next'):
if not all([[
str(loaded_models[0].model_next.weights[i]) == str(
loaded_model.model_next.weights[i])
for i in range(len(loaded_models[0].model_next.weights))
] for loaded_model in loaded_models]):
raise AssertionError('Not all model_nexts have the same weights!')
# Retrieve weights, weigh them and overwrite in model[0].
current_weights = loaded_models[0].model.get_weights()
loaded_models[0].model.set_weights([
current_weights[matrix_index] * model_weights[0]
for matrix_index in range(len(current_weights))
])
# We have model_init
if hasattr(loaded_models[0], 'model_init'):
current_weights = loaded_models[0].model_init.get_weights()
loaded_models[0].model_init.set_weights([
current_weights[matrix_index] * model_weights[0]
for matrix_index in range(len(current_weights))
])
# We have model_next
if hasattr(loaded_models[0], 'model_next'):
current_weights = loaded_models[0].model_next.get_weights()
loaded_models[0].model_next.set_weights([
current_weights[matrix_index] * model_weights[0]
for matrix_index in range(len(current_weights))
])
# Weighted sum of all models
for m in range(1, len(models)):
current_weights = loaded_models[m].model.get_weights()
prev_weights = loaded_models[0].model.get_weights()
loaded_models[0].model.set_weights([
current_weights[matrix_index] * model_weights[m] +
prev_weights[matrix_index]
for matrix_index in range(len(current_weights))
])
# We have model_init
if hasattr(loaded_models[0], 'model_init'):
current_weights = loaded_models[m].model_init.get_weights()
prev_weights = loaded_models[0].model_init.get_weights()
loaded_models[0].model_init.set_weights([
current_weights[matrix_index] * model_weights[m] +
prev_weights[matrix_index]
for matrix_index in range(len(current_weights))
])
# We have model_next
if hasattr(loaded_models[0], 'model_next'):
current_weights = loaded_models[m].model_next.get_weights()
prev_weights = loaded_models[0].model_next.get_weights()
loaded_models[0].model_next.set_weights([
current_weights[matrix_index] * model_weights[m] +
prev_weights[matrix_index]
for matrix_index in range(len(current_weights))
])
# Save averaged model
saveModel(loaded_models[0],
-1,
path=output_model,
full_path=True,
store_iter=False)
def evaluate(self, epoch, counter_name='epoch'):
# Evaluate on each set separately
for s in self.set_name:
# Apply model predictions
if self.beam_search:
params_prediction = {
'batch_size': self.batch_size,
'n_parallel_loaders':
self.extra_vars['n_parallel_loaders'],
'predict_on_sets': [s],
'pos_unk': False,
'heuristic': 0,
'mapping': None
}
params_prediction.update(
checkDefaultParamsBeamSearch(self.extra_vars))
predictions = self.model_to_eval.predictBeamSearchNet(
self.ds, params_prediction)[s]
else:
orig_size = self.extra_vars.get('eval_orig_size', False)
params_prediction = {
'batch_size': self.batch_size,
'n_parallel_loaders':
self.extra_vars['n_parallel_loaders'],
'predict_on_sets': [s]
}
# Convert predictions
postprocess_fun = None
if self.is_3DLabel:
postprocess_fun = [
self.ds.convert_3DLabels_to_bboxes,
self.extra_vars[s]['references_orig_sizes']
]
elif orig_size:
postprocess_fun = [
self.ds.resize_semantic_output,
self.extra_vars[s]['eval_orig_size_id']
]
predictions = \
self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s]
if self.is_text:
if params_prediction.get('pos_unk', False):
samples = predictions[0]
alphas = predictions[1]
if eval('self.ds.loaded_raw_' + s + '[0]'):
sources = predictions[2]
else:
sources = []
for preds in predictions[2]:
for src in preds[self.input_text_id]:
sources.append(src)
sources = decode_predictions_beam_search(
sources,
self.index2word_x,
pad_sequences=True,
verbose=self.verbose)
heuristic = params_prediction['heuristic']
else:
samples = predictions
alphas = None
heuristic = None
sources = None
if self.out_pred_idx is not None:
samples = samples[self.out_pred_idx]
# Convert predictions into sentences
if self.beam_search:
predictions = decode_predictions_beam_search(
samples,
self.index2word_y,
alphas=alphas,
x_text=sources,
heuristic=heuristic,
mapping=params_prediction['mapping'],
verbose=self.verbose)
else:
predictions = decode_predictions(
predictions,
1, # always set temperature to 1
self.index2word_y,
self.sampling_type,
verbose=self.verbose)
# Store predictions
if self.write_samples:
# Store result
filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(
epoch) + '.pred' # results file
if self.write_type == 'list':
list2file(filepath, predictions)
elif self.write_type == 'vqa':
list2vqa(filepath, predictions,
self.extra_vars[s]['question_ids'])
elif self.write_type == 'listoflists':
listoflists2file(filepath, predictions)
elif self.write_type == 'numpy':
numpy2file(filepath, predictions)
elif self.write_type == '3DLabels':
# TODO:
print("WRITE SAMPLES FUNCTION NOT IMPLEMENTED")
else:
raise NotImplementedError('The store type "' +
self.write_type +
'" is not implemented.')
# Evaluate on each metric
for metric in self.metric_name:
if self.verbose > 0:
logging.info('Evaluating on metric ' + metric)
filepath = self.save_path + '/' + s + '.' + metric # results file
# Evaluate on the chosen metric
metrics = evaluation.select[metric](pred_list=predictions,
verbose=self.verbose,
extra_vars=self.extra_vars,
split=s)
# Print results to file and store in model log
with open(filepath, 'a') as f:
header = counter_name + ','
line = str(epoch) + ','
# Store in model log
self.model_to_eval.log(s, counter_name, epoch)
for metric_ in sorted(metrics):
value = metrics[metric_]
header += metric_ + ', '
line += str(value) + ', '
# Store in model log
self.model_to_eval.log(s, metric_, value)
if epoch == 1 or epoch == self.start_eval_on_epoch:
f.write(header + '\n')
f.write(line + '\n')
if self.verbose > 0:
logging.info('Done evaluating on metric ' + metric)
# Save the model
if self.save_each_evaluation:
from keras_wrapper.cnn_model import saveModel
saveModel(self.model_to_eval,
epoch,
store_iter=not self.eval_on_epochs)
def evaluate(self, epoch, counter_name='epoch', logs=None):
if logs is None:
logs = {}
# Change inputs and outputs mappings for evaluation
self.changeInOutMappings()
# Evaluate on each set separately
all_metrics = []
for s in self.set_name:
# Apply model predictions
if self.beam_search:
params_prediction = {'max_batch_size': self.batch_size,
'n_parallel_loaders': self.extra_vars[
'n_parallel_loaders'],
'predict_on_sets': [s],
'beam_batch_size': self.beam_batch_size if
self.beam_batch_size is not None else self.batch_size,
'pos_unk': False,
'normalize': self.normalize,
'normalization_type': self.normalization_type,
'max_eval_samples': self.max_eval_samples
}
params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars))
predictions_all = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction)[s]
else:
orig_size = self.extra_vars.get('eval_orig_size', False)
params_prediction = {'batch_size': self.batch_size,
'n_parallel_loaders': self.extra_vars.get(
'n_parallel_loaders', 8),
'predict_on_sets': [s],
'normalize': self.normalize,
'normalization_type': self.normalization_type,
'max_eval_samples': self.max_eval_samples,
'model_name': self.model_name,
}
# Convert predictions
postprocess_fun = None
if self.is_3DLabel:
postprocess_fun = [self.ds.convert_3DLabels_to_bboxes,
self.extra_vars[s]['references_orig_sizes']]
elif orig_size:
postprocess_fun = [self.ds.resize_semantic_output,
self.extra_vars[s]['eval_orig_size_id']]
predictions_all = \
self.model_to_eval.predictNet(self.ds, params_prediction,
postprocess_fun=postprocess_fun)[s]
# Single-output model
if not self.gt_pos or self.gt_pos == 0 or len(self.gt_pos) == 1:
if len(predictions_all) != 2:
predictions_all = [predictions_all]
gt_positions = [0]
# Multi-output model
else:
gt_positions = self.gt_pos
# Select each output to evaluate separately
for gt_pos, type, these_metrics, gt_id, write_type, index2word_y, index2word_x in zip(
gt_positions,
self.output_types,
self.metric_name,
self.gt_id,
self.write_type,
self.index2word_y,
self.index2word_x):
predictions = predictions_all[gt_pos]
if self.verbose > 0:
print('')
logging.info('Prediction output ' + str(gt_pos) + ': ' + str(
gt_id) + ' (' + str(type) + ')')
# Postprocess outputs of type text
if type == 'text':
if params_prediction.get('pos_unk', False):
samples = predictions[0]
alphas = predictions[1]
if eval('self.ds.loaded_raw_' + s + '[0]'):
sources = predictions[2]
else:
sources = []
for preds in predictions[2]:
for src in preds[self.input_text_id]:
sources.append(src)
sources = decode_predictions_beam_search(sources,
index2word_x,
pad_sequences=True,
verbose=self.verbose)
heuristic = self.extra_vars['heuristic']
else:
samples = predictions
alphas = None
heuristic = None
sources = None
if self.out_pred_idx is not None:
samples = samples[self.out_pred_idx]
# Convert predictions into sentences
if self.beam_search:
predictions = decode_predictions_beam_search(samples,
index2word_y,
alphas=alphas,
x_text=sources,
heuristic=heuristic,
mapping=self.extra_vars.get(
'mapping',
None),
verbose=self.verbose)
else:
probs = predictions
predictions = decode_predictions(predictions,
1,
# always set temperature to 1
index2word_y,
self.sampling_type,
verbose=self.verbose)
# Apply detokenization function if needed
if self.extra_vars.get('apply_detokenization', False):
predictions = map(self.extra_vars['detokenize_f'],
predictions)
# Postprocess outputs of type binary
elif type == 'binary':
predictions = decode_multilabel(predictions,
index2word_y,
min_val=self.min_pred_multilabel[
gt_pos],
verbose=self.verbose)
# Prepare references
# exec ("y_raw = self.ds.Y_" + s + "[gt_id]")
y_split = getattr(self.ds, 'Y_' + s)
y_raw = y_split[gt_id]
self.extra_vars[gt_pos][s]['references'] = self.ds.loadBinary(y_raw, gt_id)
# Postprocess outputs of type 3DLabel
elif type == '3DLabel':
self.extra_vars[gt_pos][s] = dict()
# exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]')
y_split = getattr(self.ds, 'Y_' + s)
ref = y_split[gt_id]
[ref, original_sizes] = self.ds.convert_GT_3DLabels_to_bboxes(
ref)
self.extra_vars[gt_pos][s]['references'] = ref
self.extra_vars[gt_pos][s]['references_orig_sizes'] = original_sizes
# Postprocess outputs of type 3DSemanticLabel
elif type == '3DSemanticLabel':
self.extra_vars[gt_pos]['eval_orig_size'] = self.eval_orig_size
self.extra_vars[gt_pos][s] = dict()
# exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]')
y_split = getattr(self.ds, 'Y_' + s)
ref = y_split[gt_id]
if self.eval_orig_size:
old_crop = copy.deepcopy(self.ds.img_size_crop)
self.ds.img_size_crop = copy.deepcopy(self.ds.img_size)
self.extra_vars[gt_pos][s]['eval_orig_size_id'] = np.array([gt_id] * len(ref))
ref = self.ds.load_GT_3DSemanticLabels(ref, gt_id)
if self.eval_orig_size:
self.ds.img_size_crop = copy.deepcopy(old_crop)
self.extra_vars[gt_pos][s]['references'] = ref
# Other output data types
else:
# exec ("self.extra_vars[gt_pos][s]['references'] = self.ds.Y_" + s + "[gt_id]")
y_split = getattr(self.ds, 'Y_' + s)
self.extra_vars[gt_pos][s]['references'] = y_split[gt_id]
# Store predictions
if self.write_samples:
# Store result
filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(epoch) + '_output_' + str(gt_pos) + '.pred' # results file
if write_type == 'list':
list2file(filepath, predictions)
elif write_type == 'vqa':
try:
# exec ('refs = self.ds.Y_' + s + '[gt_id]')
y_split = getattr(self.ds, 'Y_' + s)
refs = y_split[gt_id]
except Exception:
refs = ['N/A' for _ in range(probs.shape[0])]
extra_data_plot = {'reference': refs,
'probs': probs,
'vocab': index2word_y}
list2vqa(filepath, predictions,
self.extra_vars[gt_pos][s]['question_ids'],
extra=extra_data_plot)
elif write_type == 'listoflists':
listoflists2file(filepath, predictions)
elif write_type == 'numpy':
numpy2file(filepath, predictions)
elif write_type == '3DLabels':
raise NotImplementedError(
'Write 3DLabels function is not implemented')
elif write_type == '3DSemanticLabel':
folder_path = self.save_path + '/' + s + '_' + counter_name + '_' + str(
epoch) # results folder
numpy2imgs(folder_path,
predictions,
eval('self.ds.X_' + s + '["' + self.input_id + '"]'),
self.ds)
else:
raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.')
# Evaluate on each metric
for metric in these_metrics:
if self.verbose > 0:
logging.info('Evaluating on metric ' + metric)
filepath = self.save_path + '/' + s + '.' + metric # results file
if s == 'train':
logging.info(
"WARNING: evaluation results on 'train' split might be incorrect when"
"applying random image shuffling.")
# Evaluate on the chosen metric
metrics = evaluation.select[metric](
pred_list=predictions,
verbose=self.verbose,
extra_vars=self.extra_vars[gt_pos],
split=s)
# Print results to file and store in model log
with open(filepath, 'a') as f:
header = counter_name + ','
line = str(epoch) + ','
# Store in model log
self.model_to_eval.log(s, counter_name, epoch)
for metric_ in sorted(metrics):
value = metrics[metric_]
# Multiple-output model
if self.gt_pos and self.gt_pos != 0:
metric_ += '_output_' + str(gt_pos)
all_metrics.append(metric_)
header += metric_ + ','
line += str(value) + ','
# Store in model log
self.model_to_eval.log(s, metric_, value)
if not self.written_header:
f.write(header + '\n')
self.written_header = True
f.write(line + '\n')
if self.verbose > 0:
logging.info('Done evaluating on metric ' + metric)
# Store losses
if logs.get('loss') is not None:
self.model_to_eval.log('train', 'train_loss', logs['loss'])
if logs.get('valid_loss') is not None:
self.model_to_eval.log('val', 'val_loss', logs['valid_loss'])
# Plot results so far
if self.do_plot:
if self.metric_name:
self.model_to_eval.plot(counter_name, set(all_metrics),
self.set_name, upperbound=self.max_plot)
# Save the model
if self.save_each_evaluation:
from keras_wrapper.cnn_model import saveModel
saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs)
# Recover inputs and outputs mappings for resume training
self.recoverInOutMappings()
def classifyFood101():
from keras_wrapper.cnn_model import CNN_Model, loadModel, saveModel
logging.info('Defining CNN model and training it.')
# Load food classification dataset
dataset_name = 'Food101'
ds = loadDataset('Datasets/Dataset_' + dataset_name + '.pkl')
# The network we are going to use needs an image input size of [224,224,3]
# for this reason we have to communicate this to the dataset instance in charge of loading the data
ds.img_size_crop['images'] = [224, 224, 3]
# Create VGG model and load weights
model_name = 'VGG_16_FunctionalAPI'
net = CNN_Model(
type='VGG_16_FunctionalAPI',
model_name=model_name,
input_shape=[224, 224, 3],
weights_path='/media/HDD_2TB/CNN_MODELS/VGG/vgg16_weights.h5',
seq_to_functional=True
) # we are setting the weights of a Sequential model into a FunctionalAPI one
# Reformat net output layer for the number of classes in our dataset
n_classes = len(ds.classes['labels'])
vis_input = net.model.get_layer('vis_input').output # input layer
drop = net.model.get_layer('last_dropout').output # layer before final FC
output = Dense(n_classes, activation='softmax',
name='output')(drop) # redefine FC-softmax layer
net.model = Model(input=vis_input,
output=output) # define inputs and outputs
# Compile
net.setOptimizer(lr=0.001, metrics=['accuracy'])
# Define the inputs and outputs mapping from our Dataset instance to our CNN_Model instance
# set input and output mappings from dataset to network
pos_images = ds.types_inputs.index('image')
pos_labels = ds.types_outputs.index('categorical')
# the first input of our dataset (pos_images) will also be the first input of our model (named vis_input)
inputMapping = {'vis_input': pos_images}
net.setInputsMapping(inputMapping)
# the first output of our dataset (pos_labels) will also be the first output of our model (named output)
outputMapping = {'output': pos_labels}
net.setOutputsMapping(outputMapping, acc_output='output')
# Save model
saveModel(net, 0)
# Load model
net = loadModel('Models/' + model_name, 0)
# the model must be compiled again when loaded
net.setOptimizer(lr=0.001, metrics=['accuracy'])
# Apply short training (1 epoch)
# training_params = {'n_epochs': 1, 'batch_size': 50,
# 'lr_decay': 2, 'lr_gamma': 0.8,
# 'epochs_for_save': 1, 'verbose': 1, 'eval_on_sets': ['val']}
# net.trainNet(ds, training_params)
# Test network on test set
test_params = {'batch_size': 50}
# net.testNet(ds, test_params)
# Predict network on all sets
test_params['predict_on_sets'] = ['val']
predictions = net.predictNet(ds, test_params)
logging.info("Predicted %d samples." % (len(predictions)))
logging.info("Done")
def classifyFood101():
from keras_wrapper.cnn_model import CNN_Model, loadModel, saveModel
logging.info('Defining CNN model and training it.')
# Load food classification dataset
dataset_name = 'Food101'
ds = loadDataset('Datasets/Dataset_' + dataset_name + '.pkl')
# The network we are going to use needs an image input size of [224,224,3]
# for this reason we have to communicate this to the dataset instance in charge of loading the data
ds.img_size_crop['images'] = [224, 224, 3]
# Create VGG model and load weights
model_name = 'VGG_16_FunctionalAPI'
net = CNN_Model(type='VGG_16_FunctionalAPI', model_name=model_name, input_shape=[224, 224, 3],
weights_path='/media/HDD_2TB/CNN_MODELS/VGG/vgg16_weights.h5',
seq_to_functional=True) # we are setting the weights of a Sequential model into a FunctionalAPI one
# Reformat net output layer for the number of classes in our dataset
n_classes = len(ds.classes['labels'])
vis_input = net.model.get_layer('vis_input').output # input layer
drop = net.model.get_layer('last_dropout').output # layer before final FC
output = Dense(n_classes, activation='softmax', name='output')(drop) # redefine FC-softmax layer
net.model = Model(input=vis_input, output=output) # define inputs and outputs
# Compile
net.setOptimizer(lr=0.001, metrics=['accuracy'])
# Define the inputs and outputs mapping from our Dataset instance to our CNN_Model instance
# set input and output mappings from dataset to network
pos_images = ds.types_inputs.index('image')
pos_labels = ds.types_outputs.index('categorical')
# the first input of our dataset (pos_images) will also be the first input of our model (named vis_input)
inputMapping = {'vis_input': pos_images}
net.setInputsMapping(inputMapping)
# the first output of our dataset (pos_labels) will also be the first output of our model (named output)
outputMapping = {'output': pos_labels}
net.setOutputsMapping(outputMapping, acc_output='output')
# Save model
saveModel(net, 0)
# Load model
net = loadModel('Models/' + model_name, 0)
# the model must be compiled again when loaded
net.setOptimizer(lr=0.001, metrics=['accuracy'])
# Apply short training (1 epoch)
# training_params = {'n_epochs': 1, 'batch_size': 50,
# 'lr_decay': 2, 'lr_gamma': 0.8,
# 'epochs_for_save': 1, 'verbose': 1, 'eval_on_sets': ['val']}
# net.trainNet(ds, training_params)
# Test network on test set
test_params = {'batch_size': 50}
# net.testNet(ds, test_params)
# Predict network on all sets
test_params['predict_on_sets'] = ['val']
predictions = net.predictNet(ds, test_params)
logging.info("Predicted %d samples." % (len(predictions)))
logging.info("Done")
def evaluate(self, epoch, counter_name='epoch', logs=None):
if logs is None:
logs = {}
# Change inputs and outputs mappings for evaluation
self.changeInOutMappings()
# Evaluate on each set separately
all_metrics = []
for s in self.set_name:
# Apply model predictions
if self.beam_search:
params_prediction = {'max_batch_size': self.batch_size,
'n_parallel_loaders': self.extra_vars.get('n_parallel_loaders', 1),
'predict_on_sets': [s],
'beam_batch_size': self.beam_batch_size if
self.beam_batch_size is not None else self.batch_size,
'pos_unk': False,
'normalize': self.normalize,
'normalization_type': self.normalization_type,
'max_eval_samples': self.max_eval_samples
}
params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars))
predictions_all = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction)[s]
else:
orig_size = self.extra_vars.get('eval_orig_size', False)
params_prediction = {'batch_size': self.batch_size,
'n_parallel_loaders': self.extra_vars.get('n_parallel_loaders', 1),
'predict_on_sets': [s],
'normalize': self.normalize,
'normalization_type': self.normalization_type,
'max_eval_samples': self.max_eval_samples,
'model_name': self.model_name,
}
# Convert predictions
postprocess_fun = None
if self.is_3DLabel:
postprocess_fun = [self.ds.convert_3DLabels_to_bboxes,
self.extra_vars[s]['references_orig_sizes']]
elif orig_size:
postprocess_fun = [self.ds.resize_semantic_output,
self.extra_vars[s]['eval_orig_size_id']]
predictions_all = \
self.model_to_eval.predictNet(self.ds, params_prediction,
postprocess_fun=postprocess_fun)[s]
# Single-output model
if not self.gt_pos or self.gt_pos == 0 or len(self.gt_pos) == 1:
if len(predictions_all) != 2:
predictions_all = [predictions_all]
gt_positions = [0]
# Multi-output model
else:
gt_positions = self.gt_pos
# Select each output to evaluate separately
for gt_pos, type_out, these_metrics, gt_id, write_type, index2word_y, index2word_x in zip(
gt_positions,
self.output_types,
self.metric_name,
self.gt_id,
self.write_type,
self.index2word_y,
self.index2word_x):
predictions = predictions_all[gt_pos]
if self.verbose > 0:
print('')
logging.info('Prediction output ' + str(gt_pos) + ': ' + str(
gt_id) + ' (' + str(type_out) + ')')
# Postprocess outputs of type text
if type_out == 'text':
if params_prediction.get('pos_unk', False):
samples = predictions[0]
alphas = predictions[1]
if eval('self.ds.loaded_raw_' + s + '[0]'):
sources = predictions[2]
else:
sources = []
for preds in predictions[2]:
for src in preds[self.input_text_id]:
sources.append(src)
sources = decode_predictions_beam_search(sources,
index2word_x,
pad_sequences=True,
verbose=self.verbose)
heuristic = self.extra_vars['heuristic']
else:
samples = predictions
alphas = None
heuristic = None
sources = None
if self.out_pred_idx is not None:
samples = samples[self.out_pred_idx]
# Convert predictions into sentences
if self.beam_search:
predictions = decode_predictions_beam_search(samples,
index2word_y,
glossary=self.extra_vars.get('glossary', None),
alphas=alphas,
x_text=sources,
heuristic=heuristic,
mapping=self.extra_vars.get('mapping', None),
verbose=self.verbose)
else:
probs = predictions
predictions = decode_predictions(predictions,
1,
# always set temperature to 1
index2word_y,
self.sampling_type,
verbose=self.verbose)
# Apply detokenization function if needed
if self.extra_vars.get('apply_detokenization', False):
predictions = list(map(self.extra_vars['detokenize_f'], predictions))
# Postprocess outputs of type binary
elif type_out == 'binary':
predictions = decode_multilabel(predictions,
index2word_y,
min_val=self.min_pred_multilabel[
gt_pos],
verbose=self.verbose)
# Prepare references
# exec ("y_raw = self.ds.Y_" + s + "[gt_id]")
y_split = getattr(self.ds, 'Y_' + s)
y_raw = y_split[gt_id]
self.extra_vars[gt_pos][s]['references'] = self.ds.loadBinary(y_raw, gt_id)
# Postprocess outputs of type 3DLabel
elif type_out == '3DLabel':
self.extra_vars[gt_pos][s] = dict()
# exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]')
y_split = getattr(self.ds, 'Y_' + s)
ref = y_split[gt_id]
[ref, original_sizes] = self.ds.convert_GT_3DLabels_to_bboxes(
ref)
self.extra_vars[gt_pos][s]['references'] = ref
self.extra_vars[gt_pos][s]['references_orig_sizes'] = original_sizes
# Postprocess outputs of type 3DSemanticLabel
elif type_out == '3DSemanticLabel':
self.extra_vars[gt_pos]['eval_orig_size'] = self.eval_orig_size
self.extra_vars[gt_pos][s] = dict()
# exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]')
y_split = getattr(self.ds, 'Y_' + s)
ref = y_split[gt_id]
if self.eval_orig_size:
old_crop = copy.deepcopy(self.ds.img_size_crop)
self.ds.img_size_crop = copy.deepcopy(self.ds.img_size)
self.extra_vars[gt_pos][s]['eval_orig_size_id'] = np.array([gt_id] * len(ref))
ref = self.ds.load_GT_3DSemanticLabels(ref, gt_id)
if self.eval_orig_size:
self.ds.img_size_crop = copy.deepcopy(old_crop)
self.extra_vars[gt_pos][s]['references'] = ref
# Other output data types
else:
# exec ("self.extra_vars[gt_pos][s]['references'] = self.ds.Y_" + s + "[gt_id]")
y_split = getattr(self.ds, 'Y_' + s)
self.extra_vars[gt_pos][s]['references'] = y_split[gt_id]
# Store predictions
if self.write_samples:
# Store result
filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(epoch) + '_output_' + str(gt_pos) + '.pred' # results file
if write_type == 'list':
list2file(filepath, predictions)
elif write_type == 'vqa':
try:
# exec ('refs = self.ds.Y_' + s + '[gt_id]')
y_split = getattr(self.ds, 'Y_' + s)
refs = y_split[gt_id]
except Exception:
refs = ['N/A' for _ in range(probs.shape[0])]
extra_data_plot = {'reference': refs,
'probs': probs,
'vocab': index2word_y}
list2vqa(filepath, predictions,
self.extra_vars[gt_pos][s]['question_ids'],
extra=extra_data_plot)
elif write_type == 'listoflists':
listoflists2file(filepath, predictions)
elif write_type == 'numpy':
numpy2file(filepath, predictions)
elif write_type == '3DLabels':
raise NotImplementedError(
'Write 3DLabels function is not implemented')
elif write_type == '3DSemanticLabel':
folder_path = self.save_path + '/' + s + '_' + counter_name + '_' + str(
epoch) # results folder
numpy2imgs(folder_path,
predictions,
eval('self.ds.X_' + s + '["' + self.input_id + '"]'),
self.ds)
else:
raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.')
# Evaluate on each metric
for metric in these_metrics:
if self.verbose > 0:
logging.info('Evaluating on metric ' + metric)
filepath = self.save_path + '/' + s + '.' + metric # results file
if s == 'train':
logging.info(
"WARNING: evaluation results on 'train' split might be incorrect when"
"applying random image shuffling.")
# Evaluate on the chosen metric
metrics = evaluation.select[metric](
pred_list=predictions,
verbose=self.verbose,
extra_vars=self.extra_vars[gt_pos],
split=s)
# Print results to file and store in model log
with open(filepath, 'a') as f:
header = counter_name + ','
line = str(epoch) + ','
# Store in model log
self.model_to_eval.log(s, counter_name, epoch)
for metric_ in sorted(metrics):
value = metrics[metric_]
# Multiple-output model
if self.gt_pos and self.gt_pos != 0:
metric_ += '_output_' + str(gt_pos)
all_metrics.append(metric_)
header += metric_ + ','
line += str(value) + ','
# Store in model log
self.model_to_eval.log(s, metric_, value)
if not self.written_header:
f.write(header + '\n')
self.written_header = True
f.write(line + '\n')
if self.verbose > 0:
logging.info('Done evaluating on metric ' + metric)
# Store losses
if logs.get('loss') is not None:
self.model_to_eval.log('train', 'train_loss', logs['loss'])
if logs.get('valid_loss') is not None:
self.model_to_eval.log('val', 'val_loss', logs['valid_loss'])
# Plot results so far
if self.do_plot:
if self.metric_name:
self.model_to_eval.plot(counter_name, set(all_metrics),
self.set_name, upperbound=self.max_plot)
# Save the model
if self.save_each_evaluation:
from keras_wrapper.cnn_model import saveModel
saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs)
# Recover inputs and outputs mappings for resume training
self.recoverInOutMappings()
def train_model(params):
"""
Main function
"""
if(params['RELOAD'] > 0):
logging.info('Resuming training.')
check_params(params)
########### Load data
dataset = build_dataset(params)
# Keep original images size if IMAGE_CROPPING == False
if not params['IMAGE_CROPPING']:
dataset.img_size_crop = dataset.img_size
###########
########### Build model
if(params['RELOAD'] == 0): # build new model
cnn_model = Segmentation_Model(params, type=params['MODEL_TYPE'], verbose=params['VERBOSE'],
model_name=params['MODEL_NAME'],
store_path=params['STORE_PATH'])
# Define the inputs and outputs mapping from our Dataset instance to our model
cnn_model.setInputsMapping(params['INPUTS_MAPPING'])
cnn_model.setOutputsMapping(params['OUTPUTS_MAPPING'])
# Save initial untrained model and try to load it again
saveModel(cnn_model, 0)
cnn_model=loadModel(params['STORE_PATH'], 0,
custom_objects={"AttentionComplex": AttentionComplex, 'WeightedMerge': WeightedMerge})
cnn_model.params = params
cnn_model.setOptimizer()
else: # resume from previously trained model
cnn_model = loadModel(params['STORE_PATH'], params['RELOAD'],
custom_objects={"AttentionComplex": AttentionComplex})
cnn_model.model_path = params['STORE_PATH']
cnn_model.params = params
cnn_model.setOptimizer()
###########
# Test model save/load
saveModel(cnn_model, 0)
cnn_model = loadModel(params['STORE_PATH'], 0,
custom_objects={"AttentionComplex": AttentionComplex})
cnn_model.setOptimizer()
########### Callbacks
callbacks = buildCallbacks(params, cnn_model, dataset)
###########
########### Training
total_start_time = timer()
logger.debug('Starting training!')
training_params = {'n_epochs': params['MAX_EPOCH'], 'batch_size': params['BATCH_SIZE'],
'lr_decay': params['LR_DECAY'], 'lr_gamma': params['LR_GAMMA'],
'epochs_for_save': params['EPOCHS_FOR_SAVE'], 'verbose': params['VERBOSE'],
'eval_on_sets': params['EVAL_ON_SETS_KERAS'], 'n_parallel_loaders': params['PARALLEL_LOADERS'],
'extra_callbacks': callbacks, 'reload_epoch': params['RELOAD'], 'epoch_offset': params['RELOAD'],
'data_augmentation': params['DATA_AUGMENTATION'], 'shuffle': params['SHUFFLE_TRAIN'],
'patience': params['PATIENCE'], 'metric_check': params['STOP_METRIC'], 'patience_check_split': params['PATIENCE_SPLIT'],
'normalize': params['NORMALIZE'], 'normalization_type': params['NORMALIZATION_TYPE'], 'mean_substraction': params['MEAN_SUBSTRACTION'],
'class_weights': params['OUTPUTS_IDS_DATASET'][0] if params['DISCARD_CLASSES'] or params['WEIGHT_CLASSES'] else None,
}
cnn_model.trainNet(dataset, training_params)
total_end_time = timer()
time_difference = total_end_time - total_start_time
logging.info('In total is {0:.2f}s = {1:.2f}m'.format(time_difference, time_difference / 60.0))
