def train_model(model, dataset, look_ahead, max_epochs, batch_size, save_model_path=None, trainer_status=None): start_training_time = t.time() if trainer_status is None: trainer_status = { "best_valid_error": np.inf, "best_epoch": 0, "epoch": 0, "nb_of_epocs_without_improvement": 0 } print '\n### Training NADE ###' while (trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead): trainer_status["epoch"] += 1 print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size) print '\tTraining ...', start_time = t.time() nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size)) train_err = 0 for index in range(nb_iterations): #current_iteration = ((trainer_status["epoch"] - 1) * nb_iterations) + index #train_err += model.learn(index, current_iteration) train_err += model.learn(index) # print train_err print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format( train_err / nb_iterations) print '\tValidating ...', start_time = t.time() valid_err, valid_err_std = get_mean_error_and_std( model, model.valid_log_prob, batch_size) print utils.get_done_text(start_time), " NLL: {0:.6f}".format( valid_err) if valid_err < trainer_status["best_valid_error"]: trainer_status["best_valid_error"] = valid_err trainer_status["best_epoch"] = trainer_status["epoch"] trainer_status["nb_of_epocs_without_improvement"] = 0 # Save best model if save_model_path is not None: save_model_params(model, save_model_path) utils.save_dict_to_json_file( os.path.join(save_model_path, "trainer_status"), trainer_status) else: trainer_status["nb_of_epocs_without_improvement"] += 1 print "### Training", utils.get_done_text(start_training_time), "###" total_train_time = t.time() - start_training_time return trainer_status["best_epoch"], total_train_time
def save(self, path): if not os.path.isdir(path): os.makedirs(path) hyperparameters = {'input_size': self.input_size, 'output_size': self.output_size} save_dict_to_json_file(pjoin(path, "meta.json"), {"name": self.__class__.__name__}) save_dict_to_json_file(pjoin(path, "hyperparams.json"), hyperparameters) params = {param_name: param.get_value() for param_name, param in self.parameters.items()} np.savez(pjoin(path, "params.npz"), **params)
def train_model(model, dataset, look_ahead, max_epochs, batch_size, save_model_path=None, trainer_status=None): start_training_time = t.time() if trainer_status is None: trainer_status = { "best_valid_error": np.inf, "best_epoch": 0, "epoch": 0, "nb_of_epocs_without_improvement": 0 } print '\n### Training NADE ###' while(trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead): trainer_status["epoch"] += 1 print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size) print '\tTraining ...', start_time = t.time() nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size)) train_err = 0 for index in range(nb_iterations): #current_iteration = ((trainer_status["epoch"] - 1) * nb_iterations) + index #train_err += model.learn(index, current_iteration) train_err += model.learn(index) # print train_err print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(train_err / nb_iterations) print '\tValidating ...', start_time = t.time() valid_err, valid_err_std = get_mean_error_and_std(model, model.valid_log_prob, batch_size) print utils.get_done_text(start_time), " NLL: {0:.6f}".format(valid_err) if valid_err < trainer_status["best_valid_error"]: trainer_status["best_valid_error"] = valid_err trainer_status["best_epoch"] = trainer_status["epoch"] trainer_status["nb_of_epocs_without_improvement"] = 0 # Save best model if save_model_path is not None: save_model_params(model, save_model_path) utils.save_dict_to_json_file(os.path.join(save_model_path, "trainer_status"), trainer_status) else: trainer_status["nb_of_epocs_without_improvement"] += 1 print "### Training", utils.get_done_text(start_training_time), "###" total_train_time = t.time() - start_training_time return trainer_status["best_epoch"], total_train_time
def train_model(model, dataset, look_ahead, shuffle_mask, nb_shuffle_per_valid, max_epochs, batch_size, shuffling_type, save_model_path=None, trainer_status=None): start_training_time = t.time() if trainer_status is None: trainer_status = { "nb_shuffles": 0, "best_valid_error": np.inf, "best_epoch": 0, "epoch": 0, "nb_of_epocs_without_improvement": 0 } # Always do a first shuffle so that the natural order does not gives us an edge model.shuffle("Full") # Reseting the mask to where they were when saved for i in range(trainer_status["nb_shuffles"]): model.shuffle(shuffling_type) print '\n### Training MADE ###' while (trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead): trainer_status["epoch"] += 1 print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size) print '\tTraining ...', start_time = t.time() nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size)) train_err = 0 for index in range(nb_iterations): train_err += model.learn(index, True) if shuffle_mask: if trainer_status["nb_shuffles"] == shuffle_mask: trainer_status["nb_shuffles"] = 0 model.reset(shuffling_type) else: model.shuffle(shuffling_type) trainer_status["nb_shuffles"] += 1 print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format( train_err / nb_iterations) print '\tValidating ...', start_time = t.time() if shuffle_mask > 0: model.reset(shuffling_type) valid_err, valid_err_std = get_mean_error_and_std( model, model.valid_log_prob, dataset['valid']['length'], shuffle_mask, shuffling_type, nb_shuffle_per_valid) if shuffle_mask > 0: model.reset(shuffling_type, trainer_status["nb_shuffles"]) print utils.get_done_text(start_time), " NLL: {0:.6f}".format( valid_err) if valid_err < trainer_status["best_valid_error"]: trainer_status["best_valid_error"] = valid_err trainer_status["best_epoch"] = trainer_status["epoch"] trainer_status["nb_of_epocs_without_improvement"] = 0 # Save best model if save_model_path is not None: save_model_params(model, save_model_path) utils.save_dict_to_json_file( os.path.join(save_model_path, "trainer_status"), trainer_status) else: trainer_status["nb_of_epocs_without_improvement"] += 1 print "### Training", utils.get_done_text(start_training_time), "###" total_train_time = t.time() - start_training_time return trainer_status["best_epoch"], total_train_time
print "### Resuming experiment ({0}). ###\n".format( experiment_name) loaded_hyperparams = utils.load_dict_from_json_file( os.path.join(save_path_experiment, "hyperparams")) loaded_trainingparams = utils.load_dict_from_json_file( os.path.join(save_path_experiment, "trainingparams")) if loaded_trainingparams != trainingparams or loaded_hyperparams != hyperparams: print "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting." exit() resume_mode = True else: os.makedirs(save_path_experiment) utils.save_dict_to_json_file( os.path.join(save_path_experiment, "hyperparams"), hyperparams) utils.save_dict_to_json_file( os.path.join(save_path_experiment, "trainingparams"), trainingparams) # # LOAD DATASET #### dataset = Dataset.get(dataset_name) if trainingparams['batch_size'] == -1: trainingparams['batch_size'] = dataset['train']['length'] # # INITIALIZING LEARNER #### if trainingparams['pre_training']: model = build_model_layer_pretraining( dataset, trainingparams, hyperparams,
def train_model(model, dataset, look_ahead, shuffle_mask, nb_shuffle_per_valid, max_epochs, batch_size, shuffling_type, save_model_path=None, trainer_status=None): start_training_time = t.time() if trainer_status is None: trainer_status = { "nb_shuffles": 0, "best_valid_error": np.inf, "best_epoch": 0, "epoch": 0, "nb_of_epocs_without_improvement": 0 } # Always do a first shuffle so that the natural order does not gives us an edge model.shuffle("Full") # Reseting the mask to where they were when saved for i in range(trainer_status["nb_shuffles"]): model.shuffle(shuffling_type) print '\n### Training MADE ###' while(trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead): trainer_status["epoch"] += 1 print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size) print '\tTraining ...', start_time = t.time() nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size)) train_err = 0 for index in range(nb_iterations): train_err += model.learn(index, True) if shuffle_mask: if trainer_status["nb_shuffles"] == shuffle_mask: trainer_status["nb_shuffles"] = 0 model.reset(shuffling_type) else: model.shuffle(shuffling_type) trainer_status["nb_shuffles"] += 1 print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(train_err / nb_iterations) print '\tValidating ...', start_time = t.time() if shuffle_mask > 0: model.reset(shuffling_type) valid_err, valid_err_std = get_mean_error_and_std(model, model.valid_log_prob, dataset['valid']['length'], shuffle_mask, shuffling_type, nb_shuffle_per_valid) if shuffle_mask > 0: model.reset(shuffling_type, trainer_status["nb_shuffles"]) print utils.get_done_text(start_time), " NLL: {0:.6f}".format(valid_err) if valid_err < trainer_status["best_valid_error"]: trainer_status["best_valid_error"] = valid_err trainer_status["best_epoch"] = trainer_status["epoch"] trainer_status["nb_of_epocs_without_improvement"] = 0 # Save best model if save_model_path is not None: save_model_params(model, save_model_path) utils.save_dict_to_json_file(os.path.join(save_model_path, "trainer_status"), trainer_status) else: trainer_status["nb_of_epocs_without_improvement"] += 1 print "### Training", utils.get_done_text(start_training_time), "###" total_train_time = t.time() - start_training_time return trainer_status["best_epoch"], total_train_time
save_path_experiment = os.path.join('./experiments/', experiment_name) if os.path.isdir(save_path_experiment): if not args.force: print "### Resuming experiment ({0}). ###\n".format(experiment_name) loaded_hyperparams = utils.load_dict_from_json_file(os.path.join(save_path_experiment, "hyperparams")) loaded_trainingparams = utils.load_dict_from_json_file(os.path.join(save_path_experiment, "trainingparams")) if loaded_trainingparams != trainingparams or loaded_hyperparams != hyperparams: print "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting." exit() resume_mode = True else: os.makedirs(save_path_experiment) utils.save_dict_to_json_file(os.path.join(save_path_experiment, "hyperparams"), hyperparams) utils.save_dict_to_json_file(os.path.join(save_path_experiment, "trainingparams"), trainingparams) # # LOAD DATASET #### dataset = Dataset.get(dataset_name) if trainingparams['batch_size'] == -1: trainingparams['batch_size'] = dataset['train']['length'] # # INITIALIZING LEARNER #### if trainingparams['pre_training']: model = build_model_layer_pretraining(dataset, trainingparams, hyperparams, trainingparams['pre_training_max_epoc']) else: model = build_model(dataset, trainingparams, hyperparams, hyperparams['hidden_sizes'])
def _compute_AIS(model, M=100, betas=BETAS, batch_size=None, seed=1234, ais_working_dir=".", force=False): ais_results_json = pjoin(ais_working_dir, "ais_results.part.json") if batch_size is None: batch_size = M # Will be executing M AIS's runs. last_sample_chain = np.zeros((M, model.input_size), dtype=config.floatX) M_log_w_ais = np.zeros(M, dtype=np.float64) model.set_rng_seed(seed) ais_results = {} if os.path.isfile(ais_results_json) and not force: print "Resuming AIS using info from {}".format(ais_results_json) ais_results = utils.load_dict_from_json_file(ais_results_json) M_log_w_ais = ais_results['M_log_w_ais'] last_sample_chain = ais_results['last_sample_chain'] lnZ_trivial = ais_results['lnZ_trivial'] # Iterate through all AIS runs. for i in range(0, M, batch_size): if i <= ais_results.get('batch_id', -1): continue model.set_rng_seed(seed+i) actual_size = min(M - i, batch_size) print "AIS run: {}/{} (using batch size of {})".format(i, M, batch_size) ais_partial_results = _compute_AIS_samples(model, M=actual_size, betas=betas) M_log_w_ais[i:i+batch_size] = ais_partial_results['M_log_w_ais'] last_sample_chain[i:i+batch_size] = ais_partial_results['last_sample_chain'] lnZ_trivial = ais_partial_results['lnZ_trivial'] # Save partial results if os.path.isfile(ais_results_json): shutil.copy(ais_results_json, ais_results_json[:-4] + "old.json") ais_results = {'batch_id': i, 'M': M, 'batch_size': batch_size, 'last_sample_chain': last_sample_chain, 'M_log_w_ais': M_log_w_ais, 'lnZ_trivial': lnZ_trivial} utils.save_dict_to_json_file(ais_results_json, ais_results) # We compute the mean of the estimated `r_AIS` Ms = np.arange(1, M+1) log_sum_w_ais = np.logaddexp.accumulate(M_log_w_ais) logcummean_Z = log_sum_w_ais - np.log(Ms) # We compute the standard deviation of the estimated `r_AIS` logstd_AIS = np.zeros_like(M_log_w_ais) for k in Ms[1:]: m = np.max(M_log_w_ais[:k]) logstd_AIS[k-1] = np.log(np.std(np.exp(M_log_w_ais[:k]-m), ddof=1)) - np.log(np.sqrt(k)) logstd_AIS[k-1] += m logstd_AIS[0] = np.nan # Standard deviation of only one sample does not exist. # The authors report AIS error using ln(Ẑ ± 3\sigma) m = max(np.nanmax(logstd_AIS), np.nanmax(logcummean_Z)) logcumstd_Z_up = np.log(np.exp(logcummean_Z-m) + 3*np.exp(logstd_AIS-m)) + m - logcummean_Z logcumstd_Z_down = -(np.log(np.exp(logcummean_Z-m) - 3*np.exp(logstd_AIS-m)) + m) + logcummean_Z # Compute the standard deviation of ln(Z) std_lnZ = np.array([np.std(M_log_w_ais[:k], ddof=1) for k in Ms[1:]]) std_lnZ = np.r_[np.nan, std_lnZ] # Standard deviation of only one sample does not exist. return {"logcummean_Z": logcummean_Z.astype(config.floatX), "logcumstd_Z_down": logcumstd_Z_down.astype(config.floatX), "logcumstd_Z_up": logcumstd_Z_up.astype(config.floatX), "logcumstd_Z": logstd_AIS.astype(config.floatX), "M_log_w_ais": M_log_w_ais, "lnZ_trivial": lnZ_trivial, "std_lnZ": std_lnZ, "last_sample_chain": last_sample_chain, "batch_size": batch_size, "seed": seed, "nb_temperatures": len(betas), "nb_samples": M }