def _eval_valid_multiple_shuffle(self, model, nb_test_per_step):
nb_shuffles = []
avg_nll = []
x_label = 'Nb Shuffle'
y_label = 'Avg NLL'
start_time = t.time()
print "#",
steps = [10, 100, 500]
nb_test_per_step += 1
for stepIdx in range(len(steps)):
step = range_start = steps[stepIdx]
if stepIdx != 0:
range_start = steps[stepIdx - 1] * (nb_test_per_step -
1) + step
for current_nb_shuffle in range(range_start,
nb_test_per_step * step, step):
print "/{0}\\".format(current_nb_shuffle),
# TODO Fix missing variable due to change of scope
# avg_valid_nll, avg_valid_nll_std = 0, 0 # get_mean_error_and_std(model, model.valid_log_prob, validset_theano.shape.eval()[0], shuffle_mask, current_nb_shuffle)
avg_valid_nll = 0
nb_shuffles += [current_nb_shuffle]
avg_nll += [round(avg_valid_nll, 2)]
self._save_graph('{0}shuffle_test_temp'.format(step), x_label,
y_label, (nb_shuffles, avg_nll, 'purple'))
print ".",
print get_done_text(start_time)
return nb_shuffles, avg_nll
def _eval_valid_multiple_shuffle(self, model, nb_test_per_step):
nb_shuffles = []
avg_nll = []
x_label = 'Nb Shuffle'
y_label = 'Avg NLL'
start_time = t.time()
print "#",
steps = [10, 100, 500]
nb_test_per_step += 1
for stepIdx in range(len(steps)):
step = range_start = steps[stepIdx]
if stepIdx != 0:
range_start = steps[stepIdx - 1] * (nb_test_per_step - 1) + step
for current_nb_shuffle in range(range_start, nb_test_per_step * step, step):
print "/{0}\\".format(current_nb_shuffle),
# TODO Fix missing variable due to change of scope
# avg_valid_nll, avg_valid_nll_std = 0, 0 # get_mean_error_and_std(model, model.valid_log_prob, validset_theano.shape.eval()[0], shuffle_mask, current_nb_shuffle)
avg_valid_nll = 0
nb_shuffles += [current_nb_shuffle]
avg_nll += [round(avg_valid_nll, 2)]
self._save_graph('{0}shuffle_test_temp'.format(step), x_label, y_label, (nb_shuffles, avg_nll, 'purple'))
print ".",
print get_done_text(start_time)
return nb_shuffles, avg_nll
def get_model(model_path):
hyperparams = utils.load_dict_from_json_file(
os.path.join(model_path, "hyperparams"))
hyperparams['weights_initialization'] = "Zeros"
trainingparams = utils.load_dict_from_json_file(
os.path.join(model_path, "trainingparams"))
dataset_name = trainingparams['dataset_name']
if dataset_name != "binarized_mnist":
raise (Exception(
"Invalid dataset. Only model trained on MNIST supported."))
#
# LOAD DATASET ####
dataset = Dataset.get(dataset_name)
if trainingparams['batch_size'] == -1:
trainingparams['batch_size'] = dataset['train']['length']
model = build_model(dataset, trainingparams, hyperparams,
hyperparams['hidden_sizes'])
print "### Loading model | Hidden:{0} CondMask:{1} Activation:{2} ... ".format(
hyperparams['hidden_sizes'], hyperparams['use_cond_mask'],
hyperparams['hidden_activation']),
start_time = t.time()
load_model_params(model, model_path)
print utils.get_done_text(start_time), "###"
return model, dataset_name, dataset
def train_model(model,
dataset,
look_ahead,
max_epochs,
batch_size,
save_model_path=None,
trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
print '\n### Training NADE ###'
while (trainer_status["epoch"] < max_epochs
and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"],
batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
#current_iteration = ((trainer_status["epoch"] - 1) * nb_iterations) + index
#train_err += model.learn(index, current_iteration)
train_err += model.learn(index)
# print train_err
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(
train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
valid_err, valid_err_std = get_mean_error_and_std(
model, model.valid_log_prob, batch_size)
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(
valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(
os.path.join(save_model_path, "trainer_status"),
trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
def get(dataset_name):
datasets = ['adult',
'binarized_mnist',
'connect4',
'dna',
'mushrooms',
'nips',
'ocr_letters',
'rcv1',
'web']
if dataset_name not in datasets:
raise ValueError('Dataset unknown: ' + dataset_name)
print '### Loading dataset [{0}] ...'.format(dataset_name),
start_time = t.time()
raw_dataset = np.load(os.path.join("datasets", dataset_name) + ".npz")
full_dataset = {'input_size': raw_dataset['inputsize']}
trainset_theano = theano.shared(value=raw_dataset['train_data'], borrow=True)
validset_theano = theano.shared(value=raw_dataset['valid_data'], borrow=True)
testset_theano = theano.shared(value=raw_dataset['test_data'], borrow=True)
full_dataset['train'] = {'data': trainset_theano, 'length': raw_dataset['train_length']}
full_dataset['valid'] = {'data': validset_theano, 'length': raw_dataset['valid_length']}
full_dataset['test'] = {'data': testset_theano, 'length': raw_dataset['test_length']}
print "(Dim:{0} Train:{1} Valid:{2} Test:{3})".format(full_dataset['input_size'], full_dataset['train']['length'], full_dataset['valid']['length'], full_dataset['test']['length']),
print get_done_text(start_time), "###"
return full_dataset
def get(dataset_name):
# List of datasets that works with the current model ?
datasets = ['adult',
'binarized_mnist',
'connect4',
'dna',
'mushrooms',
'nips',
'ocr_letters',
'rcv1',
'rcv2_russ',
'web']
# Setup dataset env
if dataset_name not in datasets:
raise ValueError('Dataset unknown: ' + dataset_name)
mldataset = __import__('mlpython.datasets.' + dataset_name, globals(), locals(), [dataset_name], -1)
datadir = os.path.join(os.getenv("MLPYTHON_DATASET_REPO"), dataset_name)
# Verify if dataset exist and if not, download it
if(not os.path.exists(datadir)):
dataset_store.download(dataset_name)
print '### Loading dataset [{0}] ...'.format(dataset_name),
start_time = t.time()
all_data = mldataset.load(datadir, load_to_memory=True)
train_data, train_metadata = all_data['train']
if dataset_name == 'binarized_mnist' or dataset_name == 'nips':
trainset = mlpb.MLProblem(train_data, train_metadata)
else:
trainset = mlpb.SubsetFieldsProblem(train_data, train_metadata)
trainset.setup()
valid_data, valid_metadata = all_data['valid']
validset = trainset.apply_on(valid_data, valid_metadata)
test_data, test_metadata = all_data['test']
testset = trainset.apply_on(test_data, test_metadata)
# Cleaning up, packaging and theanized
full_dataset = {'input_size': trainset.metadata['input_size']}
trainset_theano = theano.shared(value=Dataset._clean(trainset), borrow=True)
validset_theano = theano.shared(value=Dataset._clean(validset), borrow=True)
testset_theano = theano.shared(value=Dataset._clean(testset), borrow=True)
full_dataset['train'] = {'data': trainset_theano, 'length': all_data['train'][1]['length']}
full_dataset['valid'] = {'data': validset_theano, 'length': all_data['valid'][1]['length']}
full_dataset['test'] = {'data': testset_theano, 'length': all_data['test'][1]['length']}
print "(Dim:{0} Train:{1} Valid:{2} Test:{3})".format(trainset.metadata['input_size'], full_dataset['train']['length'], full_dataset['valid']['length'], full_dataset['test']['length']),
print get_done_text(start_time), "###"
return full_dataset
def get_model(model_path):
hyperparams = utils.load_dict_from_json_file(os.path.join(model_path, "hyperparams"))
hyperparams['weights_initialization'] = "Zeros"
trainingparams = utils.load_dict_from_json_file(os.path.join(model_path, "trainingparams"))
dataset_name = trainingparams['dataset_name']
if dataset_name != "binarized_mnist":
raise ValueError("Invalid dataset. Only model trained on MNIST supported.")
#
# LOAD DATASET ####
dataset = Dataset.get(dataset_name)
if trainingparams['batch_size'] == -1:
trainingparams['batch_size'] = dataset['train']['length']
model = build_model(dataset, trainingparams, hyperparams, hyperparams['hidden_sizes'])
print("### Loading model | Hidden:{0} CondMask:{1} Activation:{2} ... ".format(
hyperparams['hidden_sizes'], hyperparams['use_cond_mask'], hyperparams['hidden_activation']), end=' ')
start_time = t.time()
load_model_params(model, model_path)
print(utils.get_done_text(start_time), "###")
return model, dataset_name, dataset
def train_model(model, dataset, look_ahead, max_epochs, batch_size, save_model_path=None, trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
print '\n### Training NADE ###'
while(trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
#current_iteration = ((trainer_status["epoch"] - 1) * nb_iterations) + index
#train_err += model.learn(index, current_iteration)
train_err += model.learn(index)
# print train_err
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
valid_err, valid_err_std = get_mean_error_and_std(model, model.valid_log_prob, batch_size)
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(os.path.join(save_model_path, "trainer_status"), trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
def build_model(dataset, trainingparams, hyperparams, hidden_size):
print '\n### Initializing NADE ... ',
start_time = t.time()
model = NADE(dataset,
learning_rate=trainingparams['learning_rate'],
decrease_constant=trainingparams['decrease_constant'],
hidden_size=hidden_size,
random_seed=hyperparams['random_seed'],
batch_size=trainingparams['batch_size'],
hidden_activation=activation_functions[hyperparams['hidden_activation']],
momentum=trainingparams['momentum'],
dropout_rate=trainingparams['dropout_rate'],
weights_initialization=hyperparams['weights_initialization'],
tied=hyperparams['tied'])
print utils.get_done_text(start_time), "###"
# printParams(model)
return model
def build_model(dataset, trainingparams, hyperparams, hidden_size):
print '\n### Initializing NADE ... ',
start_time = t.time()
model = NADE(dataset,
learning_rate=trainingparams['learning_rate'],
decrease_constant=trainingparams['decrease_constant'],
hidden_size=hidden_size,
random_seed=hyperparams['random_seed'],
batch_size=trainingparams['batch_size'],
hidden_activation=activation_functions[hyperparams['hidden_activation']],
momentum=trainingparams['momentum'],
dropout_rate=trainingparams['dropout_rate'],
weights_initialization=hyperparams['weights_initialization'],
tied=hyperparams['tied'])
print utils.get_done_text(start_time), "###"
# printParams(model)
return model
def get(dataset_name):
datasets = ['adult',
'binarized_mnist',
'connect4',
'dna',
'mushrooms',
'nips',
'ocr_letters-orig',
'rcv1',
'web',
'accidents',
'ad',
'baudio',
'bbc',
'bnetflix',
'book',
'c20ng',
'cr52',
'cwebkb',
'jester',
'kdd',
'msnbc',
'msweb',
'nltcs',
'plants',
'pumsb_star',
'tmovie',
'tretail',
'rectangles',
'convex',
'ocr_letters',
'caltech101',
'bmnist']
# FIXME: disabling for a while
# if dataset_name not in datasets:
# raise ValueError('Dataset unknown: ' + dataset_name)
print('### Loading dataset [{0}] ...'.format(dataset_name), end=' ')
start_time = t.time()
raw_dataset = np.load(os.path.join("datasets", dataset_name) + ".npz")
full_dataset = {'input_size': raw_dataset['inputsize']}
trainset_theano = theano.shared(value=raw_dataset['train_data'], borrow=True)
validset_theano = theano.shared(value=raw_dataset['valid_data'], borrow=True)
testset_theano = theano.shared(value=raw_dataset['test_data'], borrow=True)
full_dataset['train'] = {'data': trainset_theano, 'length': raw_dataset['train_length']}
full_dataset['valid'] = {'data': validset_theano, 'length': raw_dataset['valid_length']}
full_dataset['test'] = {'data': testset_theano, 'length': raw_dataset['test_length']}
print("(Dim:{0} Train:{1} Valid:{2} Test:{3})".format(full_dataset['input_size'], full_dataset[
'train']['length'], full_dataset['valid']['length'], full_dataset['test']['length']), end=' ')
print(get_done_text(start_time), "###")
return full_dataset
def build_model(dataset, trainingparams, hyperparams, hidden_sizes):
print '\n### Initializing MADE ... ',
start_time = t.time()
model = MADE(dataset,
learning_rate=trainingparams['learning_rate'],
decrease_constant=trainingparams['decrease_constant'],
hidden_sizes=hidden_sizes,
random_seed=hyperparams['random_seed'],
batch_size=trainingparams['batch_size'],
hidden_activation=activation_functions[hyperparams['hidden_activation']],
use_cond_mask=hyperparams['use_cond_mask'],
direct_input_connect=hyperparams['direct_input_connect'],
direct_output_connect=hyperparams['direct_output_connect'],
update_rule=trainingparams['update_rule'],
dropout_rate=trainingparams['dropout_rate'],
weights_initialization=hyperparams['weights_initialization'],
mask_distribution=hyperparams['mask_distribution'])
print utils.get_done_text(start_time), "###"
return model
def build_model(dataset, trainingparams, hyperparams, hidden_sizes):
print '\n### Initializing MADE ... ',
start_time = t.time()
model = MADE(dataset,
learning_rate=trainingparams['learning_rate'],
decrease_constant=trainingparams['decrease_constant'],
hidden_sizes=hidden_sizes,
random_seed=hyperparams['random_seed'],
batch_size=trainingparams['batch_size'],
hidden_activation=activation_functions[
hyperparams['hidden_activation']],
use_cond_mask=hyperparams['use_cond_mask'],
direct_input_connect=hyperparams['direct_input_connect'],
direct_output_connect=hyperparams['direct_output_connect'],
update_rule=trainingparams['update_rule'],
dropout_rate=trainingparams['dropout_rate'],
weights_initialization=hyperparams['weights_initialization'],
mask_distribution=hyperparams['mask_distribution'])
print utils.get_done_text(start_time), "###"
return model
def get(dataset_name):
datasets = [
'adult', 'binarized_mnist', 'connect4', 'dna', 'mushrooms', 'nips',
'ocr_letters', 'rcv1', 'web', 'simple_tree'
]
if dataset_name not in datasets:
raise ValueError('Dataset unknown: ' + dataset_name)
print '### Loading dataset [{0}] ...'.format(dataset_name),
start_time = t.time()
raw_dataset = np.load(os.path.join("datasets", dataset_name) + ".npz")
full_dataset = {'input_size': raw_dataset['inputsize']}
trainset_theano = theano.shared(value=raw_dataset['train_data'],
borrow=True)
validset_theano = theano.shared(value=raw_dataset['valid_data'],
borrow=True)
testset_theano = theano.shared(value=raw_dataset['test_data'],
borrow=True)
full_dataset['train'] = {
'data': trainset_theano,
'length': raw_dataset['train_length']
}
full_dataset['valid'] = {
'data': validset_theano,
'length': raw_dataset['valid_length']
}
full_dataset['test'] = {
'data': testset_theano,
'length': raw_dataset['test_length']
}
print "(Dim:{0} Train:{1} Valid:{2} Test:{3})".format(
full_dataset['input_size'], full_dataset['train']['length'],
full_dataset['valid']['length'], full_dataset['test']['length']),
print get_done_text(start_time), "###"
return full_dataset
def build_model(dataset, trainingparams, hyperparams, hidden_sizes):
print("\n### Initializing MADE ... ", end=" ")
start_time = t.time()
model = MADE(
dataset,
learning_rate=trainingparams["learning_rate"],
decrease_constant=trainingparams["decrease_constant"],
hidden_sizes=hidden_sizes,
random_seed=hyperparams["random_seed"],
batch_size=trainingparams["batch_size"],
hidden_activation=activation_functions[hyperparams["hidden_activation"]],
use_cond_mask=hyperparams["use_cond_mask"],
direct_input_connect=hyperparams["direct_input_connect"],
direct_output_connect=hyperparams["direct_output_connect"],
update_rule=trainingparams["update_rule"],
dropout_rate=trainingparams["dropout_rate"],
weights_initialization=hyperparams["weights_initialization"],
mask_distribution=hyperparams["mask_distribution"],
)
print(utils.get_done_text(start_time), "###")
return model
model, dataset_name, dataset = get_model(args.model_path)
for run in range(args.nb_images):
print("Image {0}".format(run))
print("### Generating {} samples ...".format(
args.nb_samples_per_row * args.nb_rows), end=' ')
name = "_samples_run{}".format(run)
start_time = t.time()
if args.change_mask:
name += "_change_mask"
samples = model.sample(args.nb_samples_per_row, 0)
for i in range(1, args.nb_rows):
samples = np.vstack([model.sample(args.nb_samples_per_row, i), samples])
else:
samples = model.sample(args.nb_samples_per_row * args.nb_rows)
print(utils.get_done_text(start_time), "###")
if args.find_nearest_neighbour:
print("### Finding neighbours ...", end=' ')
start_time = t.time()
samples_neighbours = get_nearest_neighbours(samples, dataset)
print(utils.get_done_text(start_time), "###")
save_samples(args.nb_samples_per_row, args.nb_rows, samples_neighbours,
dataset_name + name + "_neighbours", os.path.basename(args.model_path))
save_samples(args.nb_samples_per_row, args.nb_rows, samples,
dataset_name + name, os.path.basename(args.model_path))
def train_model(model,
dataset,
look_ahead,
shuffle_mask,
nb_shuffle_per_valid,
max_epochs,
batch_size,
shuffling_type,
save_model_path=None,
trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"nb_shuffles": 0,
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
# Always do a first shuffle so that the natural order does not gives us an edge
model.shuffle("Full")
# Reseting the mask to where they were when saved
for i in range(trainer_status["nb_shuffles"]):
model.shuffle(shuffling_type)
print '\n### Training MADE ###'
while (trainer_status["epoch"] < max_epochs
and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"],
batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
train_err += model.learn(index, True)
if shuffle_mask:
if trainer_status["nb_shuffles"] == shuffle_mask:
trainer_status["nb_shuffles"] = 0
model.reset(shuffling_type)
else:
model.shuffle(shuffling_type)
trainer_status["nb_shuffles"] += 1
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(
train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
if shuffle_mask > 0:
model.reset(shuffling_type)
valid_err, valid_err_std = get_mean_error_and_std(
model, model.valid_log_prob, dataset['valid']['length'],
shuffle_mask, shuffling_type, nb_shuffle_per_valid)
if shuffle_mask > 0:
model.reset(shuffling_type, trainer_status["nb_shuffles"])
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(
valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(
os.path.join(save_model_path, "trainer_status"),
trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time
for run in range(args.nb_images):
print "Image {0}".format(run)
print "### Generating {} samples ...".format(args.nb_samples_per_row *
args.nb_rows),
name = "_samples_run{}".format(run)
start_time = t.time()
if args.change_mask:
name += "_change_mask"
samples = model.sample(args.nb_samples_per_row, 0)
for i in range(1, args.nb_rows):
samples = np.vstack(
[model.sample(args.nb_samples_per_row, i), samples])
else:
samples = model.sample(args.nb_samples_per_row * args.nb_rows)
print utils.get_done_text(start_time), "###"
if args.find_nearest_neighbour:
print "### Finding neighbours ...",
start_time = t.time()
samples_neighbours = get_nearest_neighbours(samples, dataset)
print utils.get_done_text(start_time), "###"
save_samples(args.nb_samples_per_row, args.nb_rows,
samples_neighbours,
dataset_name + name + "_neighbours",
os.path.basename(args.model_path))
save_samples(args.nb_samples_per_row, args.nb_rows, samples,
dataset_name + name, os.path.basename(args.model_path))
parser.add_argument('change_mask', metavar='change_mask', help="Change mask for every row when generating samples: {%(choices)s}", type=eval, choices=[False, True])
parser.add_argument('nb_images', help="# of images to generate.", type=int, default=1)
args = parser.parse_args()
model, dataset_name, dataset = get_model(args.model_path)
for run in range(args.nb_images):
print "Image {0}".format(run)
print "### Generating {} samples ...".format(args.nb_samples_per_row * args.nb_rows),
name = "_samples_run{}".format(run)
start_time = t.time()
if args.change_mask:
name += "_change_mask"
samples = model.sample(args.nb_samples_per_row, 0)
for i in range(1, args.nb_rows):
samples = np.vstack([model.sample(args.nb_samples_per_row, i), samples])
else:
samples = model.sample(args.nb_samples_per_row * args.nb_rows)
print utils.get_done_text(start_time), "###"
if args.find_nearest_neighbour:
print "### Finding neighbours ...",
start_time = t.time()
samples_neighbours = get_nearest_neighbours(samples, dataset)
print utils.get_done_text(start_time), "###"
save_samples(args.nb_samples_per_row, args.nb_rows, samples_neighbours, dataset_name + name + "_neighbours", os.path.basename(args.model_path))
save_samples(args.nb_samples_per_row, args.nb_rows, samples, dataset_name + name, os.path.basename(args.model_path))
def train_model(model, dataset, look_ahead, shuffle_mask, nb_shuffle_per_valid, max_epochs, batch_size, shuffling_type, save_model_path=None, trainer_status=None):
start_training_time = t.time()
if trainer_status is None:
trainer_status = {
"nb_shuffles": 0,
"best_valid_error": np.inf,
"best_epoch": 0,
"epoch": 0,
"nb_of_epocs_without_improvement": 0
}
# Always do a first shuffle so that the natural order does not gives us an edge
model.shuffle("Full")
# Reseting the mask to where they were when saved
for i in range(trainer_status["nb_shuffles"]):
model.shuffle(shuffling_type)
print '\n### Training MADE ###'
while(trainer_status["epoch"] < max_epochs and trainer_status["nb_of_epocs_without_improvement"] < look_ahead):
trainer_status["epoch"] += 1
print 'Epoch {0} (Batch Size {1})'.format(trainer_status["epoch"], batch_size)
print '\tTraining ...',
start_time = t.time()
nb_iterations = int(np.ceil(dataset['train']['length'] / batch_size))
train_err = 0
for index in range(nb_iterations):
train_err += model.learn(index, True)
if shuffle_mask:
if trainer_status["nb_shuffles"] == shuffle_mask:
trainer_status["nb_shuffles"] = 0
model.reset(shuffling_type)
else:
model.shuffle(shuffling_type)
trainer_status["nb_shuffles"] += 1
print utils.get_done_text(start_time), " avg NLL: {0:.6f}".format(train_err / nb_iterations)
print '\tValidating ...',
start_time = t.time()
if shuffle_mask > 0:
model.reset(shuffling_type)
valid_err, valid_err_std = get_mean_error_and_std(model, model.valid_log_prob, dataset['valid']['length'], shuffle_mask, shuffling_type, nb_shuffle_per_valid)
if shuffle_mask > 0:
model.reset(shuffling_type, trainer_status["nb_shuffles"])
print utils.get_done_text(start_time), " NLL: {0:.6f}".format(valid_err)
if valid_err < trainer_status["best_valid_error"]:
trainer_status["best_valid_error"] = valid_err
trainer_status["best_epoch"] = trainer_status["epoch"]
trainer_status["nb_of_epocs_without_improvement"] = 0
# Save best model
if save_model_path is not None:
save_model_params(model, save_model_path)
utils.save_dict_to_json_file(os.path.join(save_model_path, "trainer_status"), trainer_status)
else:
trainer_status["nb_of_epocs_without_improvement"] += 1
print "### Training", utils.get_done_text(start_training_time), "###"
total_train_time = t.time() - start_training_time
return trainer_status["best_epoch"], total_train_time