def average_models(models, output_model, weights=None): from keras_wrapper.cnn_model import loadModel, saveModel if not isinstance(models, list): raise AssertionError('You must give a list of models to average.') if len(models) == 0: raise AssertionError( 'You provided an empty list of models to average!') model_weights = np.asarray( [1. / len(models)] * len(models), dtype=np.float32) if (weights is None) or ( weights == []) else np.asarray(weights, dtype=np.float32) if len(model_weights) != len(models): raise AssertionError( 'You must give a list of weights of the same size than the list of models.' ) loaded_models = [loadModel(m, -1, full_path=True) for m in models] # Check that all models are compatible if not all( [hasattr(loaded_model, 'model') for loaded_model in loaded_models]): raise AssertionError('Not all models have the attribute "model".') if not (all([ hasattr(loaded_model, 'model_init') for loaded_model in loaded_models ]) or all([ not hasattr(loaded_model, 'model_init') for loaded_model in loaded_models ])): raise AssertionError('Not all models have the attribute "model_init".') if not (all([ hasattr(loaded_model, 'model_next') for loaded_model in loaded_models ]) or all([ not hasattr(loaded_model, 'model_next') for loaded_model in loaded_models ])): raise AssertionError('Not all models have the attribute "model_next".') # Check all layers are the same if not (all([[ str(loaded_models[0].model.weights[i]) == str( loaded_model.model.weights[i]) for i in range(len(loaded_models[0].model.weights)) ] for loaded_model in loaded_models])): raise AssertionError('Not all models have the same weights!') if hasattr(loaded_models[0], 'model_init'): if not all([[ str(loaded_models[0].model_init.weights[i]) == str( loaded_model.model_init.weights[i]) for i in range(len(loaded_models[0].model.weights)) ] for loaded_model in loaded_models]): raise AssertionError('Not all models have the same weights!') if not all([[ str(loaded_models[0].model.weights[i]) == str( loaded_model.model.weights[i]) for i in range(len(loaded_models[0].model_init.weights)) ] for loaded_model in loaded_models]): raise AssertionError('Not all model_inits have the same weights!') if hasattr(loaded_models[0], 'model_next'): if not all([[ str(loaded_models[0].model_next.weights[i]) == str( loaded_model.model_next.weights[i]) for i in range(len(loaded_models[0].model_next.weights)) ] for loaded_model in loaded_models]): raise AssertionError('Not all model_nexts have the same weights!') # Retrieve weights, weigh them and overwrite in model[0]. current_weights = loaded_models[0].model.get_weights() loaded_models[0].model.set_weights([ current_weights[matrix_index] * model_weights[0] for matrix_index in range(len(current_weights)) ]) # We have model_init if hasattr(loaded_models[0], 'model_init'): current_weights = loaded_models[0].model_init.get_weights() loaded_models[0].model_init.set_weights([ current_weights[matrix_index] * model_weights[0] for matrix_index in range(len(current_weights)) ]) # We have model_next if hasattr(loaded_models[0], 'model_next'): current_weights = loaded_models[0].model_next.get_weights() loaded_models[0].model_next.set_weights([ current_weights[matrix_index] * model_weights[0] for matrix_index in range(len(current_weights)) ]) # Weighted sum of all models for m in range(1, len(models)): current_weights = loaded_models[m].model.get_weights() prev_weights = loaded_models[0].model.get_weights() loaded_models[0].model.set_weights([ current_weights[matrix_index] * model_weights[m] + prev_weights[matrix_index] for matrix_index in range(len(current_weights)) ]) # We have model_init if hasattr(loaded_models[0], 'model_init'): current_weights = loaded_models[m].model_init.get_weights() prev_weights = loaded_models[0].model_init.get_weights() loaded_models[0].model_init.set_weights([ current_weights[matrix_index] * model_weights[m] + prev_weights[matrix_index] for matrix_index in range(len(current_weights)) ]) # We have model_next if hasattr(loaded_models[0], 'model_next'): current_weights = loaded_models[m].model_next.get_weights() prev_weights = loaded_models[0].model_next.get_weights() loaded_models[0].model_next.set_weights([ current_weights[matrix_index] * model_weights[m] + prev_weights[matrix_index] for matrix_index in range(len(current_weights)) ]) # Save averaged model saveModel(loaded_models[0], -1, path=output_model, full_path=True, store_iter=False)
def evaluate(self, epoch, counter_name='epoch'): # Evaluate on each set separately for s in self.set_name: # Apply model predictions if self.beam_search: params_prediction = { 'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s], 'pos_unk': False, 'heuristic': 0, 'mapping': None } params_prediction.update( checkDefaultParamsBeamSearch(self.extra_vars)) predictions = self.model_to_eval.predictBeamSearchNet( self.ds, params_prediction)[s] else: orig_size = self.extra_vars.get('eval_orig_size', False) params_prediction = { 'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars['n_parallel_loaders'], 'predict_on_sets': [s] } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [ self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes'] ] elif orig_size: postprocess_fun = [ self.ds.resize_semantic_output, self.extra_vars[s]['eval_orig_size_id'] ] predictions = \ self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s] if self.is_text: if params_prediction.get('pos_unk', False): samples = predictions[0] alphas = predictions[1] if eval('self.ds.loaded_raw_' + s + '[0]'): sources = predictions[2] else: sources = [] for preds in predictions[2]: for src in preds[self.input_text_id]: sources.append(src) sources = decode_predictions_beam_search( sources, self.index2word_x, pad_sequences=True, verbose=self.verbose) heuristic = params_prediction['heuristic'] else: samples = predictions alphas = None heuristic = None sources = None if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search( samples, self.index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=params_prediction['mapping'], verbose=self.verbose) else: predictions = decode_predictions( predictions, 1, # always set temperature to 1 self.index2word_y, self.sampling_type, verbose=self.verbose) # Store predictions if self.write_samples: # Store result filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str( epoch) + '.pred' # results file if self.write_type == 'list': list2file(filepath, predictions) elif self.write_type == 'vqa': list2vqa(filepath, predictions, self.extra_vars[s]['question_ids']) elif self.write_type == 'listoflists': listoflists2file(filepath, predictions) elif self.write_type == 'numpy': numpy2file(filepath, predictions) elif self.write_type == '3DLabels': # TODO: print("WRITE SAMPLES FUNCTION NOT IMPLEMENTED") else: raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.') # Evaluate on each metric for metric in self.metric_name: if self.verbose > 0: logging.info('Evaluating on metric ' + metric) filepath = self.save_path + '/' + s + '.' + metric # results file # Evaluate on the chosen metric metrics = evaluation.select[metric](pred_list=predictions, verbose=self.verbose, extra_vars=self.extra_vars, split=s) # Print results to file and store in model log with open(filepath, 'a') as f: header = counter_name + ',' line = str(epoch) + ',' # Store in model log self.model_to_eval.log(s, counter_name, epoch) for metric_ in sorted(metrics): value = metrics[metric_] header += metric_ + ', ' line += str(value) + ', ' # Store in model log self.model_to_eval.log(s, metric_, value) if epoch == 1 or epoch == self.start_eval_on_epoch: f.write(header + '\n') f.write(line + '\n') if self.verbose > 0: logging.info('Done evaluating on metric ' + metric) # Save the model if self.save_each_evaluation: from keras_wrapper.cnn_model import saveModel saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs)
def evaluate(self, epoch, counter_name='epoch', logs=None): if logs is None: logs = {} # Change inputs and outputs mappings for evaluation self.changeInOutMappings() # Evaluate on each set separately all_metrics = [] for s in self.set_name: # Apply model predictions if self.beam_search: params_prediction = {'max_batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars[ 'n_parallel_loaders'], 'predict_on_sets': [s], 'beam_batch_size': self.beam_batch_size if self.beam_batch_size is not None else self.batch_size, 'pos_unk': False, 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples } params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars)) predictions_all = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction)[s] else: orig_size = self.extra_vars.get('eval_orig_size', False) params_prediction = {'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars.get( 'n_parallel_loaders', 8), 'predict_on_sets': [s], 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples, 'model_name': self.model_name, } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes']] elif orig_size: postprocess_fun = [self.ds.resize_semantic_output, self.extra_vars[s]['eval_orig_size_id']] predictions_all = \ self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s] # Single-output model if not self.gt_pos or self.gt_pos == 0 or len(self.gt_pos) == 1: if len(predictions_all) != 2: predictions_all = [predictions_all] gt_positions = [0] # Multi-output model else: gt_positions = self.gt_pos # Select each output to evaluate separately for gt_pos, type, these_metrics, gt_id, write_type, index2word_y, index2word_x in zip( gt_positions, self.output_types, self.metric_name, self.gt_id, self.write_type, self.index2word_y, self.index2word_x): predictions = predictions_all[gt_pos] if self.verbose > 0: print('') logging.info('Prediction output ' + str(gt_pos) + ': ' + str( gt_id) + ' (' + str(type) + ')') # Postprocess outputs of type text if type == 'text': if params_prediction.get('pos_unk', False): samples = predictions[0] alphas = predictions[1] if eval('self.ds.loaded_raw_' + s + '[0]'): sources = predictions[2] else: sources = [] for preds in predictions[2]: for src in preds[self.input_text_id]: sources.append(src) sources = decode_predictions_beam_search(sources, index2word_x, pad_sequences=True, verbose=self.verbose) heuristic = self.extra_vars['heuristic'] else: samples = predictions alphas = None heuristic = None sources = None if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search(samples, index2word_y, alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.extra_vars.get( 'mapping', None), verbose=self.verbose) else: probs = predictions predictions = decode_predictions(predictions, 1, # always set temperature to 1 index2word_y, self.sampling_type, verbose=self.verbose) # Apply detokenization function if needed if self.extra_vars.get('apply_detokenization', False): predictions = map(self.extra_vars['detokenize_f'], predictions) # Postprocess outputs of type binary elif type == 'binary': predictions = decode_multilabel(predictions, index2word_y, min_val=self.min_pred_multilabel[ gt_pos], verbose=self.verbose) # Prepare references # exec ("y_raw = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) y_raw = y_split[gt_id] self.extra_vars[gt_pos][s]['references'] = self.ds.loadBinary(y_raw, gt_id) # Postprocess outputs of type 3DLabel elif type == '3DLabel': self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] [ref, original_sizes] = self.ds.convert_GT_3DLabels_to_bboxes( ref) self.extra_vars[gt_pos][s]['references'] = ref self.extra_vars[gt_pos][s]['references_orig_sizes'] = original_sizes # Postprocess outputs of type 3DSemanticLabel elif type == '3DSemanticLabel': self.extra_vars[gt_pos]['eval_orig_size'] = self.eval_orig_size self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] if self.eval_orig_size: old_crop = copy.deepcopy(self.ds.img_size_crop) self.ds.img_size_crop = copy.deepcopy(self.ds.img_size) self.extra_vars[gt_pos][s]['eval_orig_size_id'] = np.array([gt_id] * len(ref)) ref = self.ds.load_GT_3DSemanticLabels(ref, gt_id) if self.eval_orig_size: self.ds.img_size_crop = copy.deepcopy(old_crop) self.extra_vars[gt_pos][s]['references'] = ref # Other output data types else: # exec ("self.extra_vars[gt_pos][s]['references'] = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) self.extra_vars[gt_pos][s]['references'] = y_split[gt_id] # Store predictions if self.write_samples: # Store result filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(epoch) + '_output_' + str(gt_pos) + '.pred' # results file if write_type == 'list': list2file(filepath, predictions) elif write_type == 'vqa': try: # exec ('refs = self.ds.Y_' + s + '[gt_id]') y_split = getattr(self.ds, 'Y_' + s) refs = y_split[gt_id] except Exception: refs = ['N/A' for _ in range(probs.shape[0])] extra_data_plot = {'reference': refs, 'probs': probs, 'vocab': index2word_y} list2vqa(filepath, predictions, self.extra_vars[gt_pos][s]['question_ids'], extra=extra_data_plot) elif write_type == 'listoflists': listoflists2file(filepath, predictions) elif write_type == 'numpy': numpy2file(filepath, predictions) elif write_type == '3DLabels': raise NotImplementedError( 'Write 3DLabels function is not implemented') elif write_type == '3DSemanticLabel': folder_path = self.save_path + '/' + s + '_' + counter_name + '_' + str( epoch) # results folder numpy2imgs(folder_path, predictions, eval('self.ds.X_' + s + '["' + self.input_id + '"]'), self.ds) else: raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.') # Evaluate on each metric for metric in these_metrics: if self.verbose > 0: logging.info('Evaluating on metric ' + metric) filepath = self.save_path + '/' + s + '.' + metric # results file if s == 'train': logging.info( "WARNING: evaluation results on 'train' split might be incorrect when" "applying random image shuffling.") # Evaluate on the chosen metric metrics = evaluation.select[metric]( pred_list=predictions, verbose=self.verbose, extra_vars=self.extra_vars[gt_pos], split=s) # Print results to file and store in model log with open(filepath, 'a') as f: header = counter_name + ',' line = str(epoch) + ',' # Store in model log self.model_to_eval.log(s, counter_name, epoch) for metric_ in sorted(metrics): value = metrics[metric_] # Multiple-output model if self.gt_pos and self.gt_pos != 0: metric_ += '_output_' + str(gt_pos) all_metrics.append(metric_) header += metric_ + ',' line += str(value) + ',' # Store in model log self.model_to_eval.log(s, metric_, value) if not self.written_header: f.write(header + '\n') self.written_header = True f.write(line + '\n') if self.verbose > 0: logging.info('Done evaluating on metric ' + metric) # Store losses if logs.get('loss') is not None: self.model_to_eval.log('train', 'train_loss', logs['loss']) if logs.get('valid_loss') is not None: self.model_to_eval.log('val', 'val_loss', logs['valid_loss']) # Plot results so far if self.do_plot: if self.metric_name: self.model_to_eval.plot(counter_name, set(all_metrics), self.set_name, upperbound=self.max_plot) # Save the model if self.save_each_evaluation: from keras_wrapper.cnn_model import saveModel saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs) # Recover inputs and outputs mappings for resume training self.recoverInOutMappings()
def classifyFood101(): from keras_wrapper.cnn_model import CNN_Model, loadModel, saveModel logging.info('Defining CNN model and training it.') # Load food classification dataset dataset_name = 'Food101' ds = loadDataset('Datasets/Dataset_' + dataset_name + '.pkl') # The network we are going to use needs an image input size of [224,224,3] # for this reason we have to communicate this to the dataset instance in charge of loading the data ds.img_size_crop['images'] = [224, 224, 3] # Create VGG model and load weights model_name = 'VGG_16_FunctionalAPI' net = CNN_Model( type='VGG_16_FunctionalAPI', model_name=model_name, input_shape=[224, 224, 3], weights_path='/media/HDD_2TB/CNN_MODELS/VGG/vgg16_weights.h5', seq_to_functional=True ) # we are setting the weights of a Sequential model into a FunctionalAPI one # Reformat net output layer for the number of classes in our dataset n_classes = len(ds.classes['labels']) vis_input = net.model.get_layer('vis_input').output # input layer drop = net.model.get_layer('last_dropout').output # layer before final FC output = Dense(n_classes, activation='softmax', name='output')(drop) # redefine FC-softmax layer net.model = Model(input=vis_input, output=output) # define inputs and outputs # Compile net.setOptimizer(lr=0.001, metrics=['accuracy']) # Define the inputs and outputs mapping from our Dataset instance to our CNN_Model instance # set input and output mappings from dataset to network pos_images = ds.types_inputs.index('image') pos_labels = ds.types_outputs.index('categorical') # the first input of our dataset (pos_images) will also be the first input of our model (named vis_input) inputMapping = {'vis_input': pos_images} net.setInputsMapping(inputMapping) # the first output of our dataset (pos_labels) will also be the first output of our model (named output) outputMapping = {'output': pos_labels} net.setOutputsMapping(outputMapping, acc_output='output') # Save model saveModel(net, 0) # Load model net = loadModel('Models/' + model_name, 0) # the model must be compiled again when loaded net.setOptimizer(lr=0.001, metrics=['accuracy']) # Apply short training (1 epoch) # training_params = {'n_epochs': 1, 'batch_size': 50, # 'lr_decay': 2, 'lr_gamma': 0.8, # 'epochs_for_save': 1, 'verbose': 1, 'eval_on_sets': ['val']} # net.trainNet(ds, training_params) # Test network on test set test_params = {'batch_size': 50} # net.testNet(ds, test_params) # Predict network on all sets test_params['predict_on_sets'] = ['val'] predictions = net.predictNet(ds, test_params) logging.info("Predicted %d samples." % (len(predictions))) logging.info("Done")
def classifyFood101(): from keras_wrapper.cnn_model import CNN_Model, loadModel, saveModel logging.info('Defining CNN model and training it.') # Load food classification dataset dataset_name = 'Food101' ds = loadDataset('Datasets/Dataset_' + dataset_name + '.pkl') # The network we are going to use needs an image input size of [224,224,3] # for this reason we have to communicate this to the dataset instance in charge of loading the data ds.img_size_crop['images'] = [224, 224, 3] # Create VGG model and load weights model_name = 'VGG_16_FunctionalAPI' net = CNN_Model(type='VGG_16_FunctionalAPI', model_name=model_name, input_shape=[224, 224, 3], weights_path='/media/HDD_2TB/CNN_MODELS/VGG/vgg16_weights.h5', seq_to_functional=True) # we are setting the weights of a Sequential model into a FunctionalAPI one # Reformat net output layer for the number of classes in our dataset n_classes = len(ds.classes['labels']) vis_input = net.model.get_layer('vis_input').output # input layer drop = net.model.get_layer('last_dropout').output # layer before final FC output = Dense(n_classes, activation='softmax', name='output')(drop) # redefine FC-softmax layer net.model = Model(input=vis_input, output=output) # define inputs and outputs # Compile net.setOptimizer(lr=0.001, metrics=['accuracy']) # Define the inputs and outputs mapping from our Dataset instance to our CNN_Model instance # set input and output mappings from dataset to network pos_images = ds.types_inputs.index('image') pos_labels = ds.types_outputs.index('categorical') # the first input of our dataset (pos_images) will also be the first input of our model (named vis_input) inputMapping = {'vis_input': pos_images} net.setInputsMapping(inputMapping) # the first output of our dataset (pos_labels) will also be the first output of our model (named output) outputMapping = {'output': pos_labels} net.setOutputsMapping(outputMapping, acc_output='output') # Save model saveModel(net, 0) # Load model net = loadModel('Models/' + model_name, 0) # the model must be compiled again when loaded net.setOptimizer(lr=0.001, metrics=['accuracy']) # Apply short training (1 epoch) # training_params = {'n_epochs': 1, 'batch_size': 50, # 'lr_decay': 2, 'lr_gamma': 0.8, # 'epochs_for_save': 1, 'verbose': 1, 'eval_on_sets': ['val']} # net.trainNet(ds, training_params) # Test network on test set test_params = {'batch_size': 50} # net.testNet(ds, test_params) # Predict network on all sets test_params['predict_on_sets'] = ['val'] predictions = net.predictNet(ds, test_params) logging.info("Predicted %d samples." % (len(predictions))) logging.info("Done")
def evaluate(self, epoch, counter_name='epoch', logs=None): if logs is None: logs = {} # Change inputs and outputs mappings for evaluation self.changeInOutMappings() # Evaluate on each set separately all_metrics = [] for s in self.set_name: # Apply model predictions if self.beam_search: params_prediction = {'max_batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars.get('n_parallel_loaders', 1), 'predict_on_sets': [s], 'beam_batch_size': self.beam_batch_size if self.beam_batch_size is not None else self.batch_size, 'pos_unk': False, 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples } params_prediction.update(checkDefaultParamsBeamSearch(self.extra_vars)) predictions_all = self.model_to_eval.predictBeamSearchNet(self.ds, params_prediction)[s] else: orig_size = self.extra_vars.get('eval_orig_size', False) params_prediction = {'batch_size': self.batch_size, 'n_parallel_loaders': self.extra_vars.get('n_parallel_loaders', 1), 'predict_on_sets': [s], 'normalize': self.normalize, 'normalization_type': self.normalization_type, 'max_eval_samples': self.max_eval_samples, 'model_name': self.model_name, } # Convert predictions postprocess_fun = None if self.is_3DLabel: postprocess_fun = [self.ds.convert_3DLabels_to_bboxes, self.extra_vars[s]['references_orig_sizes']] elif orig_size: postprocess_fun = [self.ds.resize_semantic_output, self.extra_vars[s]['eval_orig_size_id']] predictions_all = \ self.model_to_eval.predictNet(self.ds, params_prediction, postprocess_fun=postprocess_fun)[s] # Single-output model if not self.gt_pos or self.gt_pos == 0 or len(self.gt_pos) == 1: if len(predictions_all) != 2: predictions_all = [predictions_all] gt_positions = [0] # Multi-output model else: gt_positions = self.gt_pos # Select each output to evaluate separately for gt_pos, type_out, these_metrics, gt_id, write_type, index2word_y, index2word_x in zip( gt_positions, self.output_types, self.metric_name, self.gt_id, self.write_type, self.index2word_y, self.index2word_x): predictions = predictions_all[gt_pos] if self.verbose > 0: print('') logging.info('Prediction output ' + str(gt_pos) + ': ' + str( gt_id) + ' (' + str(type_out) + ')') # Postprocess outputs of type text if type_out == 'text': if params_prediction.get('pos_unk', False): samples = predictions[0] alphas = predictions[1] if eval('self.ds.loaded_raw_' + s + '[0]'): sources = predictions[2] else: sources = [] for preds in predictions[2]: for src in preds[self.input_text_id]: sources.append(src) sources = decode_predictions_beam_search(sources, index2word_x, pad_sequences=True, verbose=self.verbose) heuristic = self.extra_vars['heuristic'] else: samples = predictions alphas = None heuristic = None sources = None if self.out_pred_idx is not None: samples = samples[self.out_pred_idx] # Convert predictions into sentences if self.beam_search: predictions = decode_predictions_beam_search(samples, index2word_y, glossary=self.extra_vars.get('glossary', None), alphas=alphas, x_text=sources, heuristic=heuristic, mapping=self.extra_vars.get('mapping', None), verbose=self.verbose) else: probs = predictions predictions = decode_predictions(predictions, 1, # always set temperature to 1 index2word_y, self.sampling_type, verbose=self.verbose) # Apply detokenization function if needed if self.extra_vars.get('apply_detokenization', False): predictions = list(map(self.extra_vars['detokenize_f'], predictions)) # Postprocess outputs of type binary elif type_out == 'binary': predictions = decode_multilabel(predictions, index2word_y, min_val=self.min_pred_multilabel[ gt_pos], verbose=self.verbose) # Prepare references # exec ("y_raw = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) y_raw = y_split[gt_id] self.extra_vars[gt_pos][s]['references'] = self.ds.loadBinary(y_raw, gt_id) # Postprocess outputs of type 3DLabel elif type_out == '3DLabel': self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] [ref, original_sizes] = self.ds.convert_GT_3DLabels_to_bboxes( ref) self.extra_vars[gt_pos][s]['references'] = ref self.extra_vars[gt_pos][s]['references_orig_sizes'] = original_sizes # Postprocess outputs of type 3DSemanticLabel elif type_out == '3DSemanticLabel': self.extra_vars[gt_pos]['eval_orig_size'] = self.eval_orig_size self.extra_vars[gt_pos][s] = dict() # exec ('ref=self.ds.Y_' + s + '["' + gt_id + '"]') y_split = getattr(self.ds, 'Y_' + s) ref = y_split[gt_id] if self.eval_orig_size: old_crop = copy.deepcopy(self.ds.img_size_crop) self.ds.img_size_crop = copy.deepcopy(self.ds.img_size) self.extra_vars[gt_pos][s]['eval_orig_size_id'] = np.array([gt_id] * len(ref)) ref = self.ds.load_GT_3DSemanticLabels(ref, gt_id) if self.eval_orig_size: self.ds.img_size_crop = copy.deepcopy(old_crop) self.extra_vars[gt_pos][s]['references'] = ref # Other output data types else: # exec ("self.extra_vars[gt_pos][s]['references'] = self.ds.Y_" + s + "[gt_id]") y_split = getattr(self.ds, 'Y_' + s) self.extra_vars[gt_pos][s]['references'] = y_split[gt_id] # Store predictions if self.write_samples: # Store result filepath = self.save_path + '/' + s + '_' + counter_name + '_' + str(epoch) + '_output_' + str(gt_pos) + '.pred' # results file if write_type == 'list': list2file(filepath, predictions) elif write_type == 'vqa': try: # exec ('refs = self.ds.Y_' + s + '[gt_id]') y_split = getattr(self.ds, 'Y_' + s) refs = y_split[gt_id] except Exception: refs = ['N/A' for _ in range(probs.shape[0])] extra_data_plot = {'reference': refs, 'probs': probs, 'vocab': index2word_y} list2vqa(filepath, predictions, self.extra_vars[gt_pos][s]['question_ids'], extra=extra_data_plot) elif write_type == 'listoflists': listoflists2file(filepath, predictions) elif write_type == 'numpy': numpy2file(filepath, predictions) elif write_type == '3DLabels': raise NotImplementedError( 'Write 3DLabels function is not implemented') elif write_type == '3DSemanticLabel': folder_path = self.save_path + '/' + s + '_' + counter_name + '_' + str( epoch) # results folder numpy2imgs(folder_path, predictions, eval('self.ds.X_' + s + '["' + self.input_id + '"]'), self.ds) else: raise NotImplementedError('The store type "' + self.write_type + '" is not implemented.') # Evaluate on each metric for metric in these_metrics: if self.verbose > 0: logging.info('Evaluating on metric ' + metric) filepath = self.save_path + '/' + s + '.' + metric # results file if s == 'train': logging.info( "WARNING: evaluation results on 'train' split might be incorrect when" "applying random image shuffling.") # Evaluate on the chosen metric metrics = evaluation.select[metric]( pred_list=predictions, verbose=self.verbose, extra_vars=self.extra_vars[gt_pos], split=s) # Print results to file and store in model log with open(filepath, 'a') as f: header = counter_name + ',' line = str(epoch) + ',' # Store in model log self.model_to_eval.log(s, counter_name, epoch) for metric_ in sorted(metrics): value = metrics[metric_] # Multiple-output model if self.gt_pos and self.gt_pos != 0: metric_ += '_output_' + str(gt_pos) all_metrics.append(metric_) header += metric_ + ',' line += str(value) + ',' # Store in model log self.model_to_eval.log(s, metric_, value) if not self.written_header: f.write(header + '\n') self.written_header = True f.write(line + '\n') if self.verbose > 0: logging.info('Done evaluating on metric ' + metric) # Store losses if logs.get('loss') is not None: self.model_to_eval.log('train', 'train_loss', logs['loss']) if logs.get('valid_loss') is not None: self.model_to_eval.log('val', 'val_loss', logs['valid_loss']) # Plot results so far if self.do_plot: if self.metric_name: self.model_to_eval.plot(counter_name, set(all_metrics), self.set_name, upperbound=self.max_plot) # Save the model if self.save_each_evaluation: from keras_wrapper.cnn_model import saveModel saveModel(self.model_to_eval, epoch, store_iter=not self.eval_on_epochs) # Recover inputs and outputs mappings for resume training self.recoverInOutMappings()
def train_model(params): """ Main function """ if(params['RELOAD'] > 0): logging.info('Resuming training.') check_params(params) ########### Load data dataset = build_dataset(params) # Keep original images size if IMAGE_CROPPING == False if not params['IMAGE_CROPPING']: dataset.img_size_crop = dataset.img_size ########### ########### Build model if(params['RELOAD'] == 0): # build new model cnn_model = Segmentation_Model(params, type=params['MODEL_TYPE'], verbose=params['VERBOSE'], model_name=params['MODEL_NAME'], store_path=params['STORE_PATH']) # Define the inputs and outputs mapping from our Dataset instance to our model cnn_model.setInputsMapping(params['INPUTS_MAPPING']) cnn_model.setOutputsMapping(params['OUTPUTS_MAPPING']) # Save initial untrained model and try to load it again saveModel(cnn_model, 0) cnn_model=loadModel(params['STORE_PATH'], 0, custom_objects={"AttentionComplex": AttentionComplex, 'WeightedMerge': WeightedMerge}) cnn_model.params = params cnn_model.setOptimizer() else: # resume from previously trained model cnn_model = loadModel(params['STORE_PATH'], params['RELOAD'], custom_objects={"AttentionComplex": AttentionComplex}) cnn_model.model_path = params['STORE_PATH'] cnn_model.params = params cnn_model.setOptimizer() ########### # Test model save/load saveModel(cnn_model, 0) cnn_model = loadModel(params['STORE_PATH'], 0, custom_objects={"AttentionComplex": AttentionComplex}) cnn_model.setOptimizer() ########### Callbacks callbacks = buildCallbacks(params, cnn_model, dataset) ########### ########### Training total_start_time = timer() logger.debug('Starting training!') training_params = {'n_epochs': params['MAX_EPOCH'], 'batch_size': params['BATCH_SIZE'], 'lr_decay': params['LR_DECAY'], 'lr_gamma': params['LR_GAMMA'], 'epochs_for_save': params['EPOCHS_FOR_SAVE'], 'verbose': params['VERBOSE'], 'eval_on_sets': params['EVAL_ON_SETS_KERAS'], 'n_parallel_loaders': params['PARALLEL_LOADERS'], 'extra_callbacks': callbacks, 'reload_epoch': params['RELOAD'], 'epoch_offset': params['RELOAD'], 'data_augmentation': params['DATA_AUGMENTATION'], 'shuffle': params['SHUFFLE_TRAIN'], 'patience': params['PATIENCE'], 'metric_check': params['STOP_METRIC'], 'patience_check_split': params['PATIENCE_SPLIT'], 'normalize': params['NORMALIZE'], 'normalization_type': params['NORMALIZATION_TYPE'], 'mean_substraction': params['MEAN_SUBSTRACTION'], 'class_weights': params['OUTPUTS_IDS_DATASET'][0] if params['DISCARD_CLASSES'] or params['WEIGHT_CLASSES'] else None, } cnn_model.trainNet(dataset, training_params) total_end_time = timer() time_difference = total_end_time - total_start_time logging.info('In total is {0:.2f}s = {1:.2f}m'.format(time_difference, time_difference / 60.0))