def launch_adversarial_training(training_params): """ Load the data, and train a Keras model. :param training_params: a TrainingParams object which contains each parameter of the training :return: """ if os.path.exists(training_params.path_out) is False: os.mkdir(os.path.abspath(training_params.path_out)) ###### LOADING VALIDATION DATA ####### validset, valid_targets = load_dataset_in_memory_and_resize(training_params.data_access, "valid", training_params.dataset_path, training_params.targets_path, training_params.final_size, training_params.final_size, training_params.test_batch_size) valid_targets = convert_labels(valid_targets) ###### Preprocessing VALIDATION DATA ####### for mode in training_params.valid_preprocessing: validset = preprocess_dataset(validset, training_params, mode) # Transpose validset >> (N, channel, X, Y) validset = validset.transpose(0,3,1,2) # Multiple input ? if training_params.multiple_inputs>1: validset = [validset for i in range(training_params.multiple_inputs)] ###### MODEL INITIALIZATION ####### with timer("Model initialization"): model = training_params.initialize_model() if training_params.pretrained_model is not None: with timer("Pretrained Model initialization"): pretrained_model = training_params.initialize_pretrained_model() training_params.generator_args.append(pretrained_model) # preprocessed the validset if type(pretrained_model) is list: features = [] for pmodel in pretrained_model: features.append(pmodel.predict(validset)) validset = np.concatenate(features, axis=1) else: validset = pretrained_model.predict(validset) ###### SAVE PARAMS ###### s = training_params.print_params() # Save command f = open(training_params.path_out+"/command.txt", "w") f.writelines(" ".join(sys.argv)) f.writelines(s) f.close() # Print architecture print_architecture(model, path_out=training_params.path_out + "/architecture.txt") ###### TRAINING SET ####### train_dataset = FuelDataset("train", training_params.tmp_size, batch_size=training_params.batch_size, bagging=training_params.bagging_size, bagging_iterator=training_params.bagging_iterator) ###### ADVERSARIAL MAPPING ###### input_ = model.layers[0].input y_ = model.y layer_output = model.layers[-1].get_output() xent = K.categorical_crossentropy(y_, layer_output) loss = xent.mean() grads = K.gradients(loss, input_) get_grads = K.function([input_, y_], [loss, grads]) ###### TRAINING LOOP ####### count = training_params.fine_tuning epoch_count = 0 with timer("Training"): while training_params.learning_rate >= training_params.learning_rate_min and epoch_count<training_params.nb_max_epoch: if count != 0: # Restart from the best model with a lower LR model = training_params.initialize_model() model.load_weights(training_params.path_out+"/MEM_%d/best_model.cnn"%(count-1)) # Recompile get_grads input_ = model.layers[0].input y_ = model.y layer_output = model.layers[-1].get_output() xent = K.categorical_crossentropy(y_, layer_output) loss = xent.mean() grads = K.gradients(loss, input_) get_grads = K.function([input_, y_], [loss, grads]) best = 0.0 patience = training_params.max_no_best losses = [] adv_losses = [] accuracies = [] adv_accuracies = [] valid_losses = [] valid_accuracies = [] epoch_count = 0 no_best_count = 0 path = training_params.path_out + "/MEM_%d"%count if os.path.exists(path) is False: os.mkdir(path) # Log file f = open(path+"/log.txt", "w") f.write("LR = %.2f\n"%training_params.learning_rate) f.close() # Config file open(path+"/config.netconf", 'w').write(model.to_json()) while no_best_count < patience and epoch_count < training_params.nb_max_epoch: new = True loss = 0.0 adv_loss = 0.0 accuracy = 0.0 adv_accuracy = 0.0 # Trainset Loop N = training_params.Ntrain/(training_params.batch_size*1) for i in range(N): # Train print "\rEpoch %d : Batch %d over %d"%(epoch_count, i, N), processed_batch, labels = get_next_batch(train_dataset, training_params.batch_size, training_params.final_size, training_params.preprocessing_func, training_params.preprocessing_args) l, acc = model.train_on_batch(processed_batch, labels, accuracy=True) # Update stats if new: loss = l accuracy = acc else: loss = 0.9*loss + 0.1*l accuracy = 0.9*accuracy + 0.1*acc # Get adversarial examples l, grads = get_grads([processed_batch, labels]) updates = np.sign(grads) adversarials = processed_batch + updates # Train on adv examples adv_l, adv_acc = model.train_on_batch(adversarials, labels, accuracy=True) # Update stats if new: adv_loss = adv_l adv_accuracy = adv_acc new = False else: adv_loss = 0.9*adv_loss + 0.1*adv_l adv_accuracy = 0.9*adv_accuracy + 0.1*adv_acc # Store stats losses.append(loss) accuracies.append(accuracy) adv_losses.append(adv_loss) adv_accuracies.append(adv_accuracy) # Validset loss and accuracy out = model.predict(validset) valid_loss = categorical_crossentropy(valid_targets, out) count = np.sum(np.argmax(valid_targets, axis=1) - np.argmax(out, axis=1) == 0) score = float(count)/valid_targets.shape[0] valid_losses.append(valid_loss) valid_accuracies.append(score) # Stop criterion and Save model string = "***\nEpoch %d: Loss : %0.5f, Adv loss : %0.5f, Valid loss : %0.5f, " \ "Acc : %0.5f, Adv acc : %0.5f, Valid acc : %0.5f"%(epoch_count, losses[-1], adv_losses[-1], valid_losses[-1], accuracies[-1], adv_accuracies[-1], valid_accuracies[-1]) if score > best: no_best_count = 0 save_path = path+"/best_model.cnn" if training_params.verbose>0: string = string +"\tBEST\n" print string write_log(path+"/log.txt", string) best = score model.save_weights(save_path, overwrite=True) else: no_best_count += 1 save_path = path+"/last_epoch.cnn" if training_params.verbose>0: string = string + "\n" print string write_log(path+"/log.txt", string) model.save_weights(save_path, overwrite=True) epoch_count += 1 # Update learning rate training_params.learning_rate *= 0.1 training_params.update_model_args() with open(path + "/history.pkl","w") as f: pickle.dump(losses,f) pickle.dump(adv_losses,f) pickle.dump(valid_losses,f) pickle.dump(accuracies,f) pickle.dump(adv_accuracies,f) pickle.dump(valid_accuracies,f) count += 1
def launch_training(training_params): """ Load the data, and train a Keras model. :param training_params: a TrainingParams object which contains each parameter of the training :return: """ if os.path.exists(training_params.path_out) is False: os.mkdir(os.path.abspath(training_params.path_out)) ###### LOADING VALIDATION DATA ####### validset, valid_targets = load_dataset_in_memory_and_resize(training_params.data_access, "valid", training_params.division, training_params.dataset_path, training_params.targets_path, training_params.final_size, training_params.final_size, training_params.test_batch_size) valid_targets = convert_labels(valid_targets) ###### Preprocessing VALIDATION DATA ####### for mode in training_params.valid_preprocessing: validset = preprocess_dataset(validset, training_params, mode) # Transpose validset >> (N, channel, X, Y) validset = validset.transpose(0,3,1,2) # Multiple input ? if training_params.multiple_inputs>1: validset = [validset for i in range(training_params.multiple_inputs)] ###### MODEL INITIALIZATION ####### with timer("Model initialization"): model = training_params.initialize_model() if training_params.pretrained_model is not None: with timer("Pretrained Model initialization"): pretrained_model = training_params.initialize_pretrained_model() training_params.generator_args.append(pretrained_model) # preprocessed the validset if type(pretrained_model) is list: features = [] for pmodel in pretrained_model: features.append(pmodel.predict(validset)) validset = np.concatenate(features, axis=1) else: validset = pretrained_model.predict(validset) ###### SAVE PARAMS ###### s = training_params.print_params() # Save command f = open(training_params.path_out+"/command.txt", "w") f.writelines(" ".join(sys.argv)) f.writelines(s) f.close() # Print architecture print_architecture(model, path_out=training_params.path_out + "/architecture.txt") ###### TRAINING LOOP ####### count = training_params.fine_tuning with timer("Training"): while training_params.learning_rate >= training_params.learning_rate_min and count<training_params.nb_max_epoch: if count != 0: # Restart from the best model with a lower LR model = training_params.initialize_model() model.load_weights(training_params.path_out+"/MEM_%d/best_model.cnn"%(count-1)) # Callbacks early_stoping = EarlyStopping(monitor="val_loss",patience=training_params.max_no_best) save_model = ModelCheckpoint_perso(filepath=training_params.path_out+"/MEM_%d"%count, verbose=1, optional_string=s, monitor="val_acc", mode="acc") history = model.fit_generator(training_params.generator(*training_params.generator_args), nb_epoch=training_params.nb_max_epoch, samples_per_epoch= int(training_params.Ntrain*training_params.bagging_size), show_accuracy=True, verbose=training_params.verbose, validation_data=(validset, valid_targets), callbacks=[early_stoping, save_model]) training_params.learning_rate *= 0.1 training_params.update_model_args() save_history(training_params.path_out+"/MEM_%d/history.pkl"%count, history) count += 1