def _train(self, batch_size, max_epochs): print_section('Training model') patience = self.params.initial_patience # look as this many examples regardless patience_increase = self.params.patience_increase # wait this much longer when a new best is found improvement_threshold = self.params.improvement_threshold # a relative improvement of this much is considered significant learning_rate = self.params.learning_rate learning_adjustment = self.params.learning_adjustment learning_decrease = self.params.learning_decrease nr_learning_adjustments = 0 print('---- Initial learning rate {}'.format(learning_rate)) max_factor = self.params.factor_rate factor_adjustment = self.params.factor_adjustment factor_decrease = self.params.factor_decrease factor_minimum = self.params.factor_minimum print('---- Initial loss mixture ratio {}'.format(max_factor)) curriculum = self.params.curriculum_enable curriculum_start = self.params.curriculum_start curriculum_adjustment = self.params.curriculum_adjustment # go through this many minibatch before checking the network on the validation set gui_frequency = 500 validation_frequency = min(self.nr_train_batches, patience / 2) best_validation_loss = np.inf best_iter = 0 test_score = 0. self.start_time = timeit.default_timer() storage = ParamStorage() nr_chunks = self.data.get_chunk_number() epoch = 0 done_looping = False iter = 0 #==== INITIAL PERFORMANCE ==== chunk_batches = self.data.get_elements(0) / batch_size validation_score = self._get_validation_score(batch_size, epoch, 0) test_score = self._get_test_score(batch_size) training_score = self._get_training_score(chunk_batches) #==== UPDATE GUI ==== if visual_params.gui_enabled: interface.server.append_job_update(epoch, training_score, validation_score, test_score, learning_rate) try: while (epoch < max_epochs) and (not done_looping): epoch = epoch + 1 if (epoch % learning_adjustment == 0): learning_rate *= learning_decrease nr_learning_adjustments += 1 #Temp learning_adjustment = max(10, int(learning_adjustment / 2)) print('---- New learning rate {}'.format(learning_rate)) if (epoch > factor_adjustment): max_factor = max(max_factor * factor_decrease, factor_minimum) print('---- New convex combination {}'.format(max_factor)) if (epoch % 20 == 0): print('---- Storing temp model') storage.store_params(self.model.params, id=str(epoch)) if (curriculum and epoch % curriculum_adjustment == 0 and epoch >= curriculum_start): print( "---- Mixing examples from next stage with training data" ) self.data.mix_in_next_stage() #For current examples chunk in GPU memory for chunk_index in range(nr_chunks): self.data.switch_active_training_set(chunk_index) nr_elements = self.data.get_elements(chunk_index) chunk_batches = nr_elements / batch_size #Each chunk contains a certain number of batches. for minibatch_index in range(chunk_batches): cost_ij = self.train_model(minibatch_index, learning_rate, max_factor) if iter % 1000 == 0: print( '---- Training @ iter = {}. Patience = {}. Loss = {}' .format(iter, patience, cost_ij)) if visual_params.gui_enabled and iter % gui_frequency == 0: interface.server.get_command_status() if visual_params.gui_enabled and interface.server.is_testing( ): self._debug(batch_size, chunk_batches, max_factor) #if(np.isnan(cost_ij)): # print('cost IS NAN') #==== EVAULATE ==== if (iter + 1) % validation_frequency == 0: #==== CURRENT PERFORMANCE ==== validation_score = self._get_validation_score( batch_size, epoch, minibatch_index) test_score = self._get_test_score(batch_size) train_score = self._get_training_score( chunk_batches) #No other purpose than charting #==== UPDATE GUI ==== if visual_params.gui_enabled: interface.server.append_job_update( epoch, train_score, validation_score, test_score, learning_rate) self.events.append({ "epoch": epoch, "training_loss": train_score, "validation_loss": validation_score, "test_loss": test_score, "training_rate": learning_rate }) #==== EARLY STOPPING ==== if validation_score < best_validation_loss: #improve patience if loss improvement is good enough if validation_score < best_validation_loss * improvement_threshold: patience = max(patience, iter * patience_increase) print( "---- New best validation loss. Patience increased to {}" .format(patience)) # save best validation score and iteration number best_validation_loss = validation_score best_iter = iter if patience <= iter: done_looping = True break if visual_params.gui_enabled and interface.server.stop: done_looping = True iter += 1 #Increment interation after each batch has been processed. except KeyboardInterrupt: self.set_result(best_iter, iter, best_validation_loss, test_score, nr_learning_adjustments, epoch) print( "Inpterupted by user. Current model params will be saved now.") except Exception as e: print "Unexpected error:", sys.exc_info()[0] raise self.set_result(best_iter, iter, best_validation_loss, test_score, nr_learning_adjustments, epoch)
''' print_section("TOOLS: Measure precision and recall of model") print("-data: path to dataset | -store: job_gui id to store curve in GUI "\ "| -store_path: store results locally | -model: stored model to use") #====== Arguments =============================================== is_dataset_path, dataset_path = get_command( '-data', default='/home/olav/Pictures/Mass_roads_alpha') store_gui, job_id = get_command('-store_gui', default='-1') is_store_path, store_path = get_command('-store_path', default='./pr_data.json') is_model, model_path = get_command('-model', default='./results/params.pkl') #============================================================== store = ParamStorage() data = store.load_params(path=model_path) batch_size = data['optimization'].batch_size measurer = PrecisionRecallCurve(dataset_path, data['params'], data['model'], data['dataset']) datapoints = measurer.get_curves_datapoints(batch_size) if store_gui: send_precision_recall_data(datapoints, None, job_id=job_id) else: with open(store_path, 'w') as outfile: json.dump([{"curve": datapoints, "events": []}], outfile) plt.suptitle('Precision and recall') plt.xlabel('Recall')
is_teacher_location, teacher_location = get_command( '-teacher', default=filename_params.curriculum_teacher) verify, stage = get_command('-verify', default="0") stage = "stage" + stage is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5") tradeoff = float(tradeoff) #Dataset path. Config used if not supplied is_alt_dataset, alt_dataset = get_command('-dataset') if is_alt_dataset: dataset_path = alt_dataset #============================================================== store = ParamStorage() teacher = store.load_params(path=teacher_location) evaluate = util.create_simple_predictor(teacher['model'], teacher['params']) if not verify: creator = Creator(pr_path, dim=(dataset_params.input_dim, dataset_params.output_dim), preproccessing=dataset_params.use_preprocessing, std=dataset_params.dataset_std, reduce_training=dataset_params.reduce_training, reduce_testing=dataset_params.reduce_testing, reduce_validation=dataset_params.reduce_validation) creator.load_dataset() data, labels = creator.sample_data(creator.train,
verify, stage = get_command('-verify', default="0") stage = "stage" + stage is_tradeoff, tradeoff = get_command('-tradeoff', default="0.5") tradeoff = float(tradeoff) #Dataset path. Config used if not supplied is_alt_dataset, alt_dataset = get_command('-dataset') if is_alt_dataset: dataset_path = alt_dataset #============================================================== store = ParamStorage() teacher = store.load_params(path=teacher_location) evaluate = util.create_simple_predictor(teacher['model'], teacher['params']) if not verify: creator = Creator( pr_path, dim=(dataset_params.input_dim, dataset_params.output_dim), preproccessing=dataset_params.use_preprocessing, std=dataset_params.dataset_std, reduce_training=dataset_params.reduce_training, reduce_testing=dataset_params.reduce_testing, reduce_validation=dataset_params.reduce_validation ) creator.load_dataset()
""" print_section("TOOLS: Measure precision and recall of model") print( "-data: path to dataset | -store: job_gui id to store curve in GUI " "| -store_path: store results locally | -model: stored model to use" ) # ====== Arguments =============================================== is_dataset_path, dataset_path = get_command("-data", default="/home/olav/Pictures/Mass_roads_alpha") store_gui, job_id = get_command("-store_gui", default="-1") is_store_path, store_path = get_command("-store_path", default="./pr_data.json") is_model, model_path = get_command("-model", default="./results/params.pkl") # ============================================================== store = ParamStorage() data = store.load_params(path=model_path) batch_size = data["optimization"].batch_size measurer = PrecisionRecallCurve(dataset_path, data["params"], data["model"], data["dataset"]) datapoints = measurer.get_curves_datapoints(batch_size) if store_gui: send_precision_recall_data(datapoints, None, job_id=job_id) else: with open(store_path, "w") as outfile: json.dump([{"curve": datapoints, "events": []}], outfile) plt.suptitle("Precision and recall") plt.xlabel("Recall")
def _train(self, batch_size, max_epochs): print_section('Training model') patience = self.params.initial_patience # look as this many examples regardless patience_increase = self.params.patience_increase # wait this much longer when a new best is found improvement_threshold = self.params.improvement_threshold # a relative improvement of this much is considered significant learning_rate = self.params.learning_rate learning_adjustment = self.params.learning_adjustment learning_decrease = self.params.learning_decrease nr_learning_adjustments = 0 print('---- Initial learning rate {}'.format(learning_rate)) max_factor = self.params.factor_rate factor_adjustment = self.params.factor_adjustment factor_decrease = self.params.factor_decrease factor_minimum = self.params.factor_minimum print('---- Initial loss mixture ratio {}'.format(max_factor)) curriculum = self.params.curriculum_enable curriculum_start = self.params.curriculum_start curriculum_adjustment = self.params.curriculum_adjustment # go through this many minibatch before checking the network on the validation set gui_frequency = 500 validation_frequency = min(self.nr_train_batches, patience / 2) best_validation_loss = np.inf best_iter = 0 test_score = 0. self.start_time = timeit.default_timer() storage = ParamStorage() nr_chunks = self.data.get_chunk_number() epoch = 0 done_looping = False iter = 0 #==== INITIAL PERFORMANCE ==== chunk_batches = self.data.get_elements( 0 ) / batch_size validation_score = self._get_validation_score(batch_size, epoch, 0) test_score = self._get_test_score(batch_size) training_score = self._get_training_score(chunk_batches) #==== UPDATE GUI ==== if visual_params.gui_enabled: interface.server.append_job_update(epoch, training_score, validation_score, test_score, learning_rate) try: while (epoch < max_epochs) and (not done_looping): epoch = epoch + 1 if(epoch % learning_adjustment == 0): learning_rate *= learning_decrease nr_learning_adjustments += 1 #Temp learning_adjustment = max(10, int(learning_adjustment/2)) print('---- New learning rate {}'.format(learning_rate)) if(epoch > factor_adjustment): max_factor = max(max_factor * factor_decrease, factor_minimum) print('---- New convex combination {}'.format(max_factor)) if(epoch % 20 == 0): print('---- Storing temp model') storage.store_params(self.model.params, id=str(epoch)) if(curriculum and epoch % curriculum_adjustment == 0 and epoch >= curriculum_start): print("---- Mixing examples from next stage with training data") self.data.mix_in_next_stage() #For current examples chunk in GPU memory for chunk_index in range(nr_chunks): self.data.switch_active_training_set( chunk_index ) nr_elements = self.data.get_elements( chunk_index ) chunk_batches = nr_elements / batch_size #Each chunk contains a certain number of batches. for minibatch_index in range(chunk_batches): cost_ij = self.train_model(minibatch_index, learning_rate, max_factor) if iter % 1000 == 0: print('---- Training @ iter = {}. Patience = {}. Loss = {}'.format(iter, patience, cost_ij)) if visual_params.gui_enabled and iter % gui_frequency == 0: interface.server.get_command_status() if visual_params.gui_enabled and interface.server.is_testing(): self._debug(batch_size, chunk_batches, max_factor) #if(np.isnan(cost_ij)): # print('cost IS NAN') #==== EVAULATE ==== if (iter + 1) % validation_frequency == 0: #==== CURRENT PERFORMANCE ==== validation_score = self._get_validation_score(batch_size, epoch, minibatch_index) test_score = self._get_test_score(batch_size) train_score = self._get_training_score(chunk_batches) #No other purpose than charting #==== UPDATE GUI ==== if visual_params.gui_enabled: interface.server.append_job_update( epoch, train_score, validation_score, test_score, learning_rate) self.events.append({ "epoch": epoch, "training_loss": train_score, "validation_loss": validation_score, "test_loss": test_score, "training_rate": learning_rate }) #==== EARLY STOPPING ==== if validation_score < best_validation_loss: #improve patience if loss improvement is good enough if validation_score < best_validation_loss * improvement_threshold: patience = max(patience, iter * patience_increase) print("---- New best validation loss. Patience increased to {}".format(patience)) # save best validation score and iteration number best_validation_loss = validation_score best_iter = iter if patience <= iter: done_looping = True break if visual_params.gui_enabled and interface.server.stop: done_looping = True iter += 1 #Increment interation after each batch has been processed. except KeyboardInterrupt: self.set_result(best_iter, iter, best_validation_loss, test_score, nr_learning_adjustments, epoch) print("Inpterupted by user. Current model params will be saved now.") except Exception as e: print "Unexpected error:", sys.exc_info()[0] raise self.set_result(best_iter, iter, best_validation_loss, test_score, nr_learning_adjustments, epoch)