def continue_train_model(last_batch_idx,
data_stream,
energy_optimizer,
generator_optimizer,
model_config_dict,
model_test_name):
model_list = glob.glob(samples_dir +'/*.pkl')
# load parameters
model_param_dicts = unpickle(model_list[0])
generator_models = load_generator_model(min_num_gen_filters=model_config_dict['min_num_gen_filters'],
model_params_dict=model_param_dicts)
generator_function = generator_models[0]
generator_params = generator_models[1]
energy_models = load_energy_model(num_experts=model_config_dict['expert_size'],
model_params_dict=model_param_dicts)
feature_function = energy_models[0]
# norm_function = energy_models[1]
expert_function = energy_models[1]
# prior_function = energy_models[3]
energy_params = energy_models[2]
# compile functions
print 'COMPILING MODEL UPDATER'
t=time()
generator_updater, generator_optimizer_params = set_generator_update_function(energy_feature_function=feature_function,
# energy_norm_function=norm_function,
energy_expert_function=expert_function,
# energy_prior_function=prior_function,
generator_function=generator_function,
generator_params=generator_params,
generator_optimizer=generator_optimizer,
init_param_dict=model_param_dicts)
energy_updater, energy_optimizer_params = set_energy_update_function(energy_feature_function=feature_function,
# energy_norm_function=norm_function,
energy_expert_function=expert_function,
# energy_prior_function=prior_function,
generator_function=generator_function,
energy_params=energy_params,
energy_optimizer=energy_optimizer,
init_param_dict=model_param_dicts)
print '%.2f SEC '%(time()-t)
print 'COMPILING SAMPLING FUNCTION'
t=time()
sampling_function = set_sampling_function(generator_function=generator_function)
print '%.2f SEC '%(time()-t)
# set fixed hidden data for sampling
fixed_hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'],
high=model_config_dict['hidden_distribution'],
size=(model_config_dict['num_display'], model_config_dict['hidden_size'])))
print 'START TRAINING'
# for each epoch
input_energy_list = []
sample_energy_list = []
batch_count = 0
for e in xrange(model_config_dict['epochs']):
# train phase
batch_iters = data_stream.get_epoch_iterator()
# for each batch
for b, batch_data in enumerate(batch_iters):
# batch count up
batch_count += 1
if batch_count<last_batch_idx:
continue
# set update function inputs
input_data = transform(batch_data[0])
num_data = input_data.shape[0]
hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'],
high=model_config_dict['hidden_distribution'],
size=(num_data, model_config_dict['hidden_size'])))
noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape))
update_input = [hidden_data, noise_data]
update_output = generator_updater(*update_input)
entropy_weights = update_output[1].mean()
entropy_cost = update_output[2].mean()
noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape))
update_input = [input_data, hidden_data, noise_data]
update_output = energy_updater(*update_input)
input_energy = update_output[0].mean()
sample_energy = update_output[1].mean()
input_energy_list.append(input_energy)
sample_energy_list.append(sample_energy)
if batch_count%10==0:
print '================================================================'
print 'BATCH ITER #{}'.format(batch_count), model_test_name
print '================================================================'
print ' TRAIN RESULTS'
print '================================================================'
print ' input energy : ', input_energy_list[-1]
print '----------------------------------------------------------------'
print ' sample energy : ', sample_energy_list[-1]
print '----------------------------------------------------------------'
print ' entropy weight : ', entropy_weights
print '----------------------------------------------------------------'
print ' entropy cost : ', entropy_cost
print '================================================================'
if batch_count%100==0:
# sample data
sample_data = sampling_function(fixed_hidden_data)[0]
sample_data = np.asarray(sample_data)
save_as = samples_dir + '/' + model_test_name + '_SAMPLES{}.png'.format(batch_count)
color_grid_vis(inverse_transform(sample_data).transpose([0,2,3,1]), (16, 16), save_as)
np.save(file=samples_dir + '/' + model_test_name +'_input_energy',
arr=np.asarray(input_energy_list))
np.save(file=samples_dir + '/' + model_test_name +'_sample_energy',
arr=np.asarray(sample_energy_list))
save_as = samples_dir + '/' + model_test_name + '_MODEL.pkl'
save_model(tensor_params_list=generator_params[0] + generator_params[1] + energy_params + generator_optimizer_params + energy_optimizer_params,
save_to=save_as)
def train_model(data_stream, model_optimizer, model_config_dict, model_test_name):
encoder_model = set_encoder_model(model_config_dict["hidden_size"], model_config_dict["min_num_gen_filters"])
encoder_function = encoder_model[0]
encoder_parameters = encoder_model[1]
decoder_model = set_decoder_model(model_config_dict["hidden_size"], model_config_dict["min_num_eng_filters"])
decoder_function = decoder_model[0]
decoder_parameters = decoder_model[1]
# compile functions
print "COMPILING UPDATER FUNCTION"
t = time()
updater_function = set_updater_function(
encoder_function=encoder_function,
decoder_function=decoder_function,
encoder_params=encoder_parameters,
decoder_params=decoder_parameters,
optimizer=model_optimizer,
)
print "%.2f SEC " % (time() - t)
print "COMPILING SAMPLING FUNCTION"
t = time()
sampling_function = set_sampling_function(decoder_function=decoder_function)
print "%.2f SEC " % (time() - t)
# set fixed hidden data for sampling
fixed_hidden_data = floatX(np_rng.normal(size=(model_config_dict["num_display"], model_config_dict["hidden_size"])))
print "START TRAINING"
# for each epoch
recon_cost_list = []
moment_match_cost_list = []
model_cost_list = []
batch_count = 0
for e in xrange(model_config_dict["epochs"]):
# train phase
batch_iters = data_stream.get_epoch_iterator()
# for each batch
for b, batch_data in enumerate(batch_iters):
# set update function inputs
positive_visible_data = transform(batch_data[0])
negative_hidden_data = floatX(
np_rng.normal(size=(positive_visible_data.shape[0], model_config_dict["hidden_size"]))
)
moment_cost_weight = 1.0
updater_inputs = [positive_visible_data, negative_hidden_data, moment_cost_weight]
updater_outputs = updater_function(*updater_inputs)
recon_cost = updater_outputs[0].mean()
moment_match_cost = updater_outputs[1].mean()
model_cost = updater_outputs[2].mean()
recon_cost_list.append(recon_cost)
moment_match_cost_list.append(moment_match_cost)
model_cost_list.append(model_cost)
# batch count up
batch_count += 1
if batch_count % 1 == 0:
print "================================================================"
print "BATCH ITER #{}".format(batch_count), model_test_name
print "================================================================"
print " TRAIN RESULTS"
print "================================================================"
print " recon cost : ", recon_cost_list[-1]
print "----------------------------------------------------------------"
print " moment cost : ", moment_match_cost_list[-1]
print "----------------------------------------------------------------"
print " model cost : ", model_cost_list[-1]
print "================================================================"
if batch_count % 100 == 0:
# sample data
sample_data = sampling_function(fixed_hidden_data)[0]
save_as = samples_dir + "/" + model_test_name + "_SAMPLES(NEGATIVE){}.png".format(batch_count)
color_grid_vis(inverse_transform(sample_data).transpose([0, 2, 3, 1]), (16, 16), save_as)
# save costs
np.save(file=samples_dir + "/" + model_test_name + "_recon_cost", arr=np.asarray(recon_cost_list))
np.save(
file=samples_dir + "/" + model_test_name + "_moment_cost", arr=np.asarray(moment_match_cost_list)
)
np.save(file=samples_dir + "/" + model_test_name + "_model_cost", arr=np.asarray(model_cost_list))
if batch_count % 1000 == 0:
save_as = samples_dir + "/" + model_test_name + "_MODEL.pkl"
save_model(tensor_params_list=decoder_parameters, save_to=save_as)
def train_model(data_stream, energy_optimizer, generator_optimizer, model_config_dict, model_test_name):
generator_models = set_generator_model(
num_hiddens=model_config_dict["hidden_size"], min_num_gen_filters=model_config_dict["min_num_gen_filters"]
)
generator_function = generator_models[0]
generator_params = generator_models[1]
energy_models = set_energy_model(
num_experts=model_config_dict["expert_size"], min_num_eng_filters=model_config_dict["min_num_eng_filters"]
)
feature_function = energy_models[0]
# norm_function = energy_models[1]
expert_function = energy_models[1]
# prior_function = energy_models[3]
energy_params = energy_models[2]
# compile functions
print "COMPILING MODEL UPDATER"
t = time()
generator_updater = set_generator_update_function(
energy_feature_function=feature_function,
# energy_norm_function=norm_function,
energy_expert_function=expert_function,
# energy_prior_function=prior_function,
generator_function=generator_function,
generator_params=generator_params,
generator_optimizer=generator_optimizer,
)
energy_updater = set_energy_update_function(
energy_feature_function=feature_function,
# energy_norm_function=norm_function,
energy_expert_function=expert_function,
# energy_prior_function=prior_function,
generator_function=generator_function,
energy_params=energy_params,
energy_optimizer=energy_optimizer,
)
print "%.2f SEC " % (time() - t)
print "COMPILING SAMPLING FUNCTION"
t = time()
sampling_function = set_sampling_function(generator_function=generator_function)
print "%.2f SEC " % (time() - t)
# set fixed hidden data for sampling
fixed_hidden_data = floatX(
np_rng.uniform(
low=-model_config_dict["hidden_distribution"],
high=model_config_dict["hidden_distribution"],
size=(model_config_dict["num_display"], model_config_dict["hidden_size"]),
)
)
print "START TRAINING"
# for each epoch
input_energy_list = []
sample_energy_list = []
batch_count = 0
for e in xrange(model_config_dict["epochs"]):
# train phase
batch_iters = data_stream.get_epoch_iterator()
# for each batch
for b, batch_data in enumerate(batch_iters):
# set update function inputs
input_data = transform(batch_data[0])
num_data = input_data.shape[0]
hidden_data = floatX(
np_rng.uniform(
low=-model_config_dict["hidden_distribution"],
high=model_config_dict["hidden_distribution"],
size=(num_data, model_config_dict["hidden_size"]),
)
)
noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape))
update_input = [hidden_data, noise_data]
update_output = generator_updater(*update_input)
entropy_weights = update_output[1].mean()
entropy_cost = update_output[2].mean()
noise_data = floatX(np_rng.normal(scale=0.01, size=input_data.shape))
update_input = [input_data, hidden_data, noise_data]
update_output = energy_updater(*update_input)
input_energy = update_output[0].mean()
sample_energy = update_output[1].mean()
input_energy_list.append(input_energy)
sample_energy_list.append(sample_energy)
# batch count up
batch_count += 1
if batch_count % 10 == 0:
print "================================================================"
print "BATCH ITER #{}".format(batch_count), model_test_name
print "================================================================"
print " TRAIN RESULTS"
print "================================================================"
print " input energy : ", input_energy_list[-1]
print "----------------------------------------------------------------"
print " sample energy : ", sample_energy_list[-1]
print "----------------------------------------------------------------"
print " entropy weight : ", entropy_weights
print "----------------------------------------------------------------"
print " entropy cost : ", entropy_cost
print "================================================================"
if batch_count % 100 == 0:
# sample data
sample_data = sampling_function(fixed_hidden_data)[0]
sample_data = np.asarray(sample_data)
save_as = samples_dir + "/" + model_test_name + "_SAMPLES(TRAIN){}.png".format(batch_count)
color_grid_vis(inverse_transform(sample_data).transpose([0, 2, 3, 1]), (16, 16), save_as)
np.save(file=samples_dir + "/" + model_test_name + "_input_energy", arr=np.asarray(input_energy_list))
np.save(file=samples_dir + "/" + model_test_name + "_sample_energy", arr=np.asarray(sample_energy_list))
save_as = samples_dir + "/" + model_test_name + "_MODEL.pkl"
save_model(
tensor_params_list=generator_params[0] + generator_params[1] + energy_params, save_to=save_as
)
def train_model(data_stream,
energy_optimizer,
generator_optimizer,
generator_bn_optimizer,
model_config_dict,
model_test_name):
[generator_function, generator_params, generator_bn_params] = set_generator_model(model_config_dict['hidden_size'],
model_config_dict['min_num_gen_filters'])
[feature_function, energy_function, energy_params] = set_energy_model(model_config_dict['expert_size'],
model_config_dict['min_num_eng_filters'])
# compile functions
print 'COMPILING ENERGY UPDATER'
t=time()
energy_updater = set_energy_update_function(feature_function=feature_function,
energy_function=energy_function,
generator_function=generator_function,
energy_params=energy_params,
energy_optimizer=energy_optimizer)
print '%.2f SEC '%(time()-t)
print 'COMPILING GENERATOR UPDATER'
t=time()
generator_updater = set_generator_update_function(feature_function=feature_function,
energy_function=energy_function,
generator_function=generator_function,
generator_params=generator_params,
generator_bn_params=generator_bn_params,
generator_optimizer=generator_optimizer,
generator_bn_optimizer=generator_bn_optimizer)
print '%.2f SEC '%(time()-t)
print 'COMPILING SAMPLING FUNCTION'
t=time()
sampling_function = set_sampling_function(generator_function=generator_function)
print '%.2f SEC '%(time()-t)
# set fixed hidden data for sampling
fixed_hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'],
high=model_config_dict['hidden_distribution'],
size=(model_config_dict['num_display'], model_config_dict['hidden_size'])))
print 'START TRAINING'
# for each epoch
input_energy_list = []
sample_energy_list = []
batch_count = 0
for e in xrange(model_config_dict['epochs']):
# train phase
batch_iters = data_stream.get_epoch_iterator()
# for each batch
for b, batch_data in enumerate(batch_iters):
# set update function inputs
input_data = transform(batch_data[0])
num_data = input_data.shape[0]
hidden_data = floatX(np_rng.uniform(low=-model_config_dict['hidden_distribution'],
high=model_config_dict['hidden_distribution'],
size=(num_data, model_config_dict['hidden_size'])))
noise_data = floatX(np_rng.normal(scale=0.01*(0.99**int(batch_count/100)),
size=(num_data, num_channels, input_shape, input_shape)))
updater_inputs = [input_data,
hidden_data,
noise_data,
batch_count]
updater_outputs = generator_updater(*updater_inputs)
noise_data = floatX(np_rng.normal(scale=0.01*(0.99**int(batch_count/100)),
size=(num_data, num_channels, input_shape, input_shape)))
updater_inputs = [input_data,
hidden_data,
noise_data,
batch_count]
updater_outputs = energy_updater(*updater_inputs)
# get output values
input_energy = updater_outputs[0].mean()
sample_energy = updater_outputs[1].mean()
input_energy_list.append(input_energy)
sample_energy_list.append(sample_energy)
# batch count up
batch_count += 1
if batch_count%1==0:
print '================================================================'
print 'BATCH ITER #{}'.format(batch_count), model_test_name
print '================================================================'
print ' TRAIN RESULTS'
print '================================================================'
print ' input energy : ', input_energy_list[-1]
print '----------------------------------------------------------------'
print ' sample energy : ', sample_energy_list[-1]
print '================================================================'
if batch_count%1000==0:
# sample data
[sample_data_t, sample_data_f] = sampling_function(fixed_hidden_data)
sample_data_t = np.asarray(sample_data_t)
save_as = samples_dir + '/' + model_test_name + '_SAMPLES(TRAIN){}.png'.format(batch_count)
color_grid_vis(inverse_transform(sample_data_t).transpose([0,2,3,1]), (16, 16), save_as)
sample_data_f = np.asarray(sample_data_f)
save_as = samples_dir + '/' + model_test_name + '_SAMPLES(TEST){}.png'.format(batch_count)
color_grid_vis(inverse_transform(sample_data_f).transpose([0,2,3,1]), (16, 16), save_as)
np.save(file=samples_dir + '/' + model_test_name +'_input_energy',
arr=np.asarray(input_energy_list))
np.save(file=samples_dir + '/' + model_test_name +'_sample_energy',
arr=np.asarray(sample_energy_list))
save_as = samples_dir + '/' + model_test_name + '_MODEL.pkl'
save_model(tensor_params_list=generator_params + generator_bn_params + energy_params,
save_to=save_as)
def train_model(data_stream,
model_optimizer,
model_config_dict,
model_test_name):
encoder_model = set_encoder_model(model_config_dict['hidden_size'],
model_config_dict['min_num_gen_filters'])
encoder_feature_function = encoder_model[0]
encoder_mean_function = encoder_model[1]
encoder_var_function = encoder_model[2]
encoder_parameters = encoder_model[3]
decoder_model = set_decoder_model(model_config_dict['hidden_size'],
model_config_dict['min_num_eng_filters'])
decoder_function = decoder_model[0]
decoder_parameters = decoder_model[1]
# compile functions
print 'COMPILING UPDATER FUNCTION'
t=time()
updater_function = set_updater_function(encoder_feature_function=encoder_feature_function,
encoder_mean_function=encoder_mean_function,
encoder_var_function=encoder_var_function,
decoder_function=decoder_function,
encoder_params=encoder_parameters,
decoder_params=decoder_parameters,
optimizer=model_optimizer)
print '%.2f SEC '%(time()-t)
print 'COMPILING SAMPLING FUNCTION'
t=time()
sampling_function = set_sampling_function(decoder_function=decoder_function)
print '%.2f SEC '%(time()-t)
# set fixed hidden data for sampling
fixed_hidden_data = floatX(np_rng.normal(size=(model_config_dict['num_display'], model_config_dict['hidden_size'])))
print 'START TRAINING'
# for each epoch
vae_cost_list = []
recon_cost_list = []
kl_cost_list = []
moment_match_cost_list = []
batch_count = 0
for e in xrange(model_config_dict['epochs']):
# train phase
batch_iters = data_stream.get_epoch_iterator()
# for each batch
for b, batch_data in enumerate(batch_iters):
# set update function inputs
positive_visible_data = transform(batch_data[0])
positive_hidden_data = floatX(np_rng.normal(size=(positive_visible_data.shape[0], model_config_dict['hidden_size'])))
negative_hidden_data = floatX(np_rng.normal(size=(positive_visible_data.shape[0], model_config_dict['hidden_size'])))
moment_cost_weight = 1.0
updater_inputs = [positive_visible_data,
positive_hidden_data,
negative_hidden_data,
moment_cost_weight]
updater_outputs = updater_function(*updater_inputs)
vae_cost = updater_outputs[0].mean()
recon_cost = updater_outputs[1].mean()
kl_cost = updater_outputs[2].mean()
moment_match_cost = updater_outputs[3].mean()
recon_samples = updater_outputs[4]
vae_cost_list.append(vae_cost)
recon_cost_list.append(recon_cost)
kl_cost_list.append(kl_cost)
moment_match_cost_list.append(moment_match_cost)
# batch count up
batch_count += 1
if batch_count%10==0:
print '================================================================'
print 'BATCH ITER #{}'.format(batch_count), model_test_name
print '================================================================'
print ' TRAIN RESULTS'
print '================================================================'
print ' vae cost : ', vae_cost_list[-1]
print '----------------------------------------------------------------'
print ' recon cost : ', recon_cost_list[-1]
print '----------------------------------------------------------------'
print ' kl cost : ', kl_cost_list[-1]
print '----------------------------------------------------------------'
print ' moment cost : ', moment_match_cost_list[-1]
print '================================================================'
if batch_count%500==0:
# sample data
sample_data = sampling_function(fixed_hidden_data)[0]
save_as = samples_dir + '/' + model_test_name + '_SAMPLES(NEGATIVE){}.png'.format(batch_count)
color_grid_vis(inverse_transform(sample_data).transpose([0,2,3,1]), (16, 16), save_as)
# recon data
save_as = samples_dir + '/' + model_test_name + '_ORIGINALS(POSITIVE){}.png'.format(batch_count)
color_grid_vis(inverse_transform(positive_visible_data).transpose([0,2,3,1]), (12, 12), save_as)
save_as = samples_dir + '/' + model_test_name + '_RECONSTRUCTIONS(POSITIVE){}.png'.format(batch_count)
color_grid_vis(inverse_transform(recon_samples).transpose([0,2,3,1]), (12, 12), save_as)
# save costs
np.save(file=samples_dir + '/' + model_test_name +'_vae_cost',
arr=np.asarray(vae_cost_list))
np.save(file=samples_dir + '/' + model_test_name +'_recon_cost',
arr=np.asarray(recon_cost_list))
np.save(file=samples_dir + '/' + model_test_name +'_kl_cost',
arr=np.asarray(kl_cost_list))
np.save(file=samples_dir + '/' + model_test_name +'_moment_cost',
arr=np.asarray(moment_match_cost_list))
if batch_count%1000==0:
save_as = samples_dir + '/' + model_test_name + '_MODEL.pkl'
save_model(tensor_params_list=decoder_parameters,
save_to=save_as)
