def train_model(model,
dataset,
look_ahead,
max_epochs,
batch_size,
save_model_path=None,
trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
print '\n### Training NADE ###'
while (trainer_status["epoch"] < max_epochs
and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"],
batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
#current_iteration = ((trainer_status["epoch"] - 1) * nb_iterations) + index
#train_err += model.learn(index, current_iteration)
train_err += model.learn(index)
# print train_err
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(
train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
valid_err, valid_err_std = get_mean_error_and_std(
model, model.valid_log_prob, batch_size)
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(
valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(
os.path.join(save_model_path, "trainer_status"),
trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
def save(self, path):
if not os.path.isdir(path):
os.makedirs(path)
hyperparameters = {'input_size': self.input_size,
'output_size': self.output_size}
save_dict_to_json_file(pjoin(path, "meta.json"), {"name": self.__class__.__name__})
save_dict_to_json_file(pjoin(path, "hyperparams.json"), hyperparameters)
params = {param_name: param.get_value() for param_name, param in self.parameters.items()}
np.savez(pjoin(path, "params.npz"), **params)
def train_model(model, dataset, look_ahead, max_epochs, batch_size, save_model_path=None, trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
print '\n### Training NADE ###'
while(trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
#current_iteration = ((trainer_status["epoch"] - 1) * nb_iterations) + index
#train_err += model.learn(index, current_iteration)
train_err += model.learn(index)
# print train_err
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
valid_err, valid_err_std = get_mean_error_and_std(model, model.valid_log_prob, batch_size)
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(os.path.join(save_model_path, "trainer_status"), trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
def train_model(model,
dataset,
look_ahead,
shuffle_mask,
nb_shuffle_per_valid,
max_epochs,
batch_size,
shuffling_type,
save_model_path=None,
trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"nb_shuffles": 0,
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
# Always do a first shuffle so that the natural order does not gives us an edge
model.shuffle("Full")
# Reseting the mask to where they were when saved
for i in range(trainer_status["nb_shuffles"]):
model.shuffle(shuffling_type)
print '\n### Training MADE ###'
while (trainer_status["epoch"] < max_epochs
and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"],
batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
train_err += model.learn(index, True)
if shuffle_mask:
if trainer_status["nb_shuffles"] == shuffle_mask:
trainer_status["nb_shuffles"] = 0
model.reset(shuffling_type)
else:
model.shuffle(shuffling_type)
trainer_status["nb_shuffles"] += 1
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(
train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
if shuffle_mask > 0:
model.reset(shuffling_type)
valid_err, valid_err_std = get_mean_error_and_std(
model, model.valid_log_prob, dataset['valid']['length'],
shuffle_mask, shuffling_type, nb_shuffle_per_valid)
if shuffle_mask > 0:
model.reset(shuffling_type, trainer_status["nb_shuffles"])
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(
valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(
os.path.join(save_model_path, "trainer_status"),
trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
print "### Resuming experiment ({0}). ###\n".format(
experiment_name)
loaded_hyperparams = utils.load_dict_from_json_file(
os.path.join(save_path_experiment, "hyperparams"))
loaded_trainingparams = utils.load_dict_from_json_file(
os.path.join(save_path_experiment, "trainingparams"))
if loaded_trainingparams != trainingparams or loaded_hyperparams != hyperparams:
print "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting."
exit()
resume_mode = True
else:
os.makedirs(save_path_experiment)
utils.save_dict_to_json_file(
os.path.join(save_path_experiment, "hyperparams"), hyperparams)
utils.save_dict_to_json_file(
os.path.join(save_path_experiment, "trainingparams"),
trainingparams)
#
# LOAD DATASET ####
dataset = Dataset.get(dataset_name)
if trainingparams['batch_size'] == -1:
trainingparams['batch_size'] = dataset['train']['length']
#
# INITIALIZING LEARNER ####
if trainingparams['pre_training']:
model = build_model_layer_pretraining(
dataset, trainingparams, hyperparams,
def train_model(model, dataset, look_ahead, shuffle_mask, nb_shuffle_per_valid, max_epochs, batch_size, shuffling_type, save_model_path=None, trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"nb_shuffles": 0,
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
# Always do a first shuffle so that the natural order does not gives us an edge
model.shuffle("Full")
# Reseting the mask to where they were when saved
for i in range(trainer_status["nb_shuffles"]):
model.shuffle(shuffling_type)
print '\n### Training MADE ###'
while(trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
train_err += model.learn(index, True)
if shuffle_mask:
if trainer_status["nb_shuffles"] == shuffle_mask:
trainer_status["nb_shuffles"] = 0
model.reset(shuffling_type)
else:
model.shuffle(shuffling_type)
trainer_status["nb_shuffles"] += 1
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
if shuffle_mask > 0:
model.reset(shuffling_type)
valid_err, valid_err_std = get_mean_error_and_std(model, model.valid_log_prob, dataset['valid']['length'], shuffle_mask, shuffling_type, nb_shuffle_per_valid)
if shuffle_mask > 0:
model.reset(shuffling_type, trainer_status["nb_shuffles"])
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(os.path.join(save_model_path, "trainer_status"), trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
save_path_experiment = os.path.join('./experiments/', experiment_name)
if os.path.isdir(save_path_experiment):
if not args.force:
print "### Resuming experiment ({0}). ###\n".format(experiment_name)
loaded_hyperparams = utils.load_dict_from_json_file(os.path.join(save_path_experiment, "hyperparams"))
loaded_trainingparams = utils.load_dict_from_json_file(os.path.join(save_path_experiment, "trainingparams"))
if loaded_trainingparams != trainingparams or loaded_hyperparams != hyperparams:
print "The arguments provided are different than the one saved. Use --force if you are certain.\nQuitting."
exit()
resume_mode = True
else:
os.makedirs(save_path_experiment)
utils.save_dict_to_json_file(os.path.join(save_path_experiment, "hyperparams"), hyperparams)
utils.save_dict_to_json_file(os.path.join(save_path_experiment, "trainingparams"), trainingparams)
#
# LOAD DATASET ####
dataset = Dataset.get(dataset_name)
if trainingparams['batch_size'] == -1:
trainingparams['batch_size'] = dataset['train']['length']
#
# INITIALIZING LEARNER ####
if trainingparams['pre_training']:
model = build_model_layer_pretraining(dataset, trainingparams, hyperparams, trainingparams['pre_training_max_epoc'])
else:
model = build_model(dataset, trainingparams, hyperparams, hyperparams['hidden_sizes'])
def _compute_AIS(model, M=100, betas=BETAS, batch_size=None, seed=1234, ais_working_dir=".", force=False):
ais_results_json = pjoin(ais_working_dir, "ais_results.part.json")
if batch_size is None:
batch_size = M
# Will be executing M AIS's runs.
last_sample_chain = np.zeros((M, model.input_size), dtype=config.floatX)
M_log_w_ais = np.zeros(M, dtype=np.float64)
model.set_rng_seed(seed)
ais_results = {}
if os.path.isfile(ais_results_json) and not force:
print "Resuming AIS using info from {}".format(ais_results_json)
ais_results = utils.load_dict_from_json_file(ais_results_json)
M_log_w_ais = ais_results['M_log_w_ais']
last_sample_chain = ais_results['last_sample_chain']
lnZ_trivial = ais_results['lnZ_trivial']
# Iterate through all AIS runs.
for i in range(0, M, batch_size):
if i <= ais_results.get('batch_id', -1):
continue
model.set_rng_seed(seed+i)
actual_size = min(M - i, batch_size)
print "AIS run: {}/{} (using batch size of {})".format(i, M, batch_size)
ais_partial_results = _compute_AIS_samples(model, M=actual_size, betas=betas)
M_log_w_ais[i:i+batch_size] = ais_partial_results['M_log_w_ais']
last_sample_chain[i:i+batch_size] = ais_partial_results['last_sample_chain']
lnZ_trivial = ais_partial_results['lnZ_trivial']
# Save partial results
if os.path.isfile(ais_results_json):
shutil.copy(ais_results_json, ais_results_json[:-4] + "old.json")
ais_results = {'batch_id': i,
'M': M,
'batch_size': batch_size,
'last_sample_chain': last_sample_chain,
'M_log_w_ais': M_log_w_ais,
'lnZ_trivial': lnZ_trivial}
utils.save_dict_to_json_file(ais_results_json, ais_results)
# We compute the mean of the estimated `r_AIS`
Ms = np.arange(1, M+1)
log_sum_w_ais = np.logaddexp.accumulate(M_log_w_ais)
logcummean_Z = log_sum_w_ais - np.log(Ms)
# We compute the standard deviation of the estimated `r_AIS`
logstd_AIS = np.zeros_like(M_log_w_ais)
for k in Ms[1:]:
m = np.max(M_log_w_ais[:k])
logstd_AIS[k-1] = np.log(np.std(np.exp(M_log_w_ais[:k]-m), ddof=1)) - np.log(np.sqrt(k))
logstd_AIS[k-1] += m
logstd_AIS[0] = np.nan # Standard deviation of only one sample does not exist.
# The authors report AIS error using ln(Ẑ ± 3\sigma)
m = max(np.nanmax(logstd_AIS), np.nanmax(logcummean_Z))
logcumstd_Z_up = np.log(np.exp(logcummean_Z-m) + 3*np.exp(logstd_AIS-m)) + m - logcummean_Z
logcumstd_Z_down = -(np.log(np.exp(logcummean_Z-m) - 3*np.exp(logstd_AIS-m)) + m) + logcummean_Z
# Compute the standard deviation of ln(Z)
std_lnZ = np.array([np.std(M_log_w_ais[:k], ddof=1) for k in Ms[1:]])
std_lnZ = np.r_[np.nan, std_lnZ] # Standard deviation of only one sample does not exist.
return {"logcummean_Z": logcummean_Z.astype(config.floatX),
"logcumstd_Z_down": logcumstd_Z_down.astype(config.floatX),
"logcumstd_Z_up": logcumstd_Z_up.astype(config.floatX),
"logcumstd_Z": logstd_AIS.astype(config.floatX),
"M_log_w_ais": M_log_w_ais,
"lnZ_trivial": lnZ_trivial,
"std_lnZ": std_lnZ,
"last_sample_chain": last_sample_chain,
"batch_size": batch_size,
"seed": seed,
"nb_temperatures": len(betas),
"nb_samples": M
}
