def pretrain_osm(lam_kld=0.0):
# Initialize a source of randomness
rng = np.random.RandomState(1234)
# Load some data to train/validate/test with
dataset = 'data/mnist.pkl.gz'
datasets = load_udm(dataset, zero_mean=False)
Xtr = datasets[0][0]
Xtr = Xtr.get_value(borrow=False)
Xva = datasets[2][0]
Xva = Xva.get_value(borrow=False)
print("Xtr.shape: {0:s}, Xva.shape: {1:s}".format(str(Xtr.shape),str(Xva.shape)))
# get and set some basic dataset information
Xtr_mean = np.mean(Xtr, axis=0)
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
batch_size = 100
batch_reps = 5
# setup some symbolic variables and stuff
Xd = T.matrix('Xd_base')
Xc = T.matrix('Xc_base')
Xm = T.matrix('Xm_base')
data_dim = Xtr.shape[1]
prior_sigma = 1.0
##########################
# NETWORK CONFIGURATIONS #
##########################
gn_params = {}
shared_config = [PRIOR_DIM, 1000, 1000]
top_config = [shared_config[-1], data_dim]
gn_params['shared_config'] = shared_config
gn_params['mu_config'] = top_config
gn_params['sigma_config'] = top_config
gn_params['activation'] = relu_actfun
gn_params['init_scale'] = 1.4
gn_params['lam_l2a'] = 0.0
gn_params['vis_drop'] = 0.0
gn_params['hid_drop'] = 0.0
gn_params['bias_noise'] = 0.0
gn_params['input_noise'] = 0.0
# choose some parameters for the continuous inferencer
in_params = {}
shared_config = [data_dim, 1000, 1000]
top_config = [shared_config[-1], PRIOR_DIM]
in_params['shared_config'] = shared_config
in_params['mu_config'] = top_config
in_params['sigma_config'] = top_config
in_params['activation'] = relu_actfun
in_params['init_scale'] = 1.4
in_params['lam_l2a'] = 0.0
in_params['vis_drop'] = 0.0
in_params['hid_drop'] = 0.0
in_params['bias_noise'] = 0.0
in_params['input_noise'] = 0.0
# Initialize the base networks for this OneStageModel
IN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=in_params, shared_param_dicts=None)
GN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=gn_params, shared_param_dicts=None)
# Initialize biases in IN and GN
IN.init_biases(0.2)
GN.init_biases(0.2)
#########################
# INITIALIZE THE GIPAIR #
#########################
osm_params = {}
osm_params['x_type'] = 'bernoulli'
osm_params['xt_transform'] = 'sigmoid'
osm_params['logvar_bound'] = LOGVAR_BOUND
OSM = OneStageModel(rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, \
p_x_given_z=GN, q_z_given_x=IN, \
x_dim=data_dim, z_dim=PRIOR_DIM, params=osm_params)
OSM.set_lam_l2w(1e-5)
safe_mean = (0.9 * Xtr_mean) + 0.05
safe_mean_logit = np.log(safe_mean / (1.0 - safe_mean))
OSM.set_output_bias(safe_mean_logit)
OSM.set_input_bias(-Xtr_mean)
######################
# BASIC VAE TRAINING #
######################
out_file = open(RESULT_PATH+"pt_osm_results.txt", 'wb')
# Set initial learning rate and basic SGD hyper parameters
obs_costs = np.zeros((batch_size,))
costs = [0. for i in range(10)]
learn_rate = 0.0005
for i in range(150000):
scale = min(1.0, float(i) / 10000.0)
if ((i > 1) and ((i % 20000) == 0)):
learn_rate = learn_rate * 0.9
# do a minibatch update of the model, and compute some costs
tr_idx = npr.randint(low=0,high=tr_samples,size=(batch_size,))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xc_batch = 0.0 * Xd_batch
Xm_batch = 0.0 * Xd_batch
# do a minibatch update of the model, and compute some costs
OSM.set_sgd_params(lr_1=(scale*learn_rate), mom_1=0.5, mom_2=0.98)
OSM.set_lam_nll(1.0)
OSM.set_lam_kld(lam_kld_1=(1.0 + (scale*(lam_kld-1.0))), lam_kld_2=0.0)
result = OSM.train_joint(Xd_batch, Xc_batch, Xm_batch, batch_reps)
costs = [(costs[j] + result[j]) for j in range(len(result))]
if ((i % 1000) == 0):
# record and then reset the cost trackers
costs = [(v / 1000.0) for v in costs]
str_1 = "-- batch {0:d} --".format(i)
str_2 = " joint_cost: {0:.4f}".format(costs[0])
str_3 = " nll_cost : {0:.4f}".format(costs[1])
str_4 = " kld_cost : {0:.4f}".format(costs[2])
str_5 = " reg_cost : {0:.4f}".format(costs[3])
costs = [0.0 for v in costs]
# print out some diagnostic information
joint_str = "\n".join([str_1, str_2, str_3, str_4, str_5])
print(joint_str)
out_file.write(joint_str+"\n")
out_file.flush()
if ((i % 2000) == 0):
Xva = row_shuffle(Xva)
model_samps = OSM.sample_from_prior(500)
file_name = RESULT_PATH+"pt_osm_samples_b{0:d}_XG.png".format(i)
utils.visualize_samples(model_samps, file_name, num_rows=20)
# compute information about free-energy on validation set
file_name = RESULT_PATH+"pt_osm_free_energy_b{0:d}.png".format(i)
fe_terms = OSM.compute_fe_terms(Xva[0:2500], 20)
fe_mean = np.mean(fe_terms[0]) + np.mean(fe_terms[1])
fe_str = " nll_bound : {0:.4f}".format(fe_mean)
print(fe_str)
out_file.write(fe_str+"\n")
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, \
x_label='Posterior KLd', y_label='Negative Log-likelihood')
# compute information about posterior KLds on validation set
file_name = RESULT_PATH+"pt_osm_post_klds_b{0:d}.png".format(i)
post_klds = OSM.compute_post_klds(Xva[0:2500])
post_dim_klds = np.mean(post_klds, axis=0)
utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
file_name)
if ((i % 5000) == 0):
IN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_b{0:d}_IN.pkl".format(i))
GN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_b{0:d}_GN.pkl".format(i))
IN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_IN.pkl")
GN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_GN.pkl")
return
def pretrain_osm(lam_kld=0.0):
# Initialize a source of randomness
rng = np.random.RandomState(1234)
# Load some data to train/validate/test with
data_file = 'data/tfd_data_48x48.pkl'
dataset = load_tfd(tfd_pkl_name=data_file, which_set='unlabeled', fold='all')
Xtr_unlabeled = dataset[0]
dataset = load_tfd(tfd_pkl_name=data_file, which_set='train', fold='all')
Xtr_train = dataset[0]
Xtr = np.vstack([Xtr_unlabeled, Xtr_train])
dataset = load_tfd(tfd_pkl_name=data_file, which_set='valid', fold='all')
Xva = dataset[0]
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
batch_size = 400
batch_reps = 6
carry_frac = 0.25
carry_size = int(batch_size * carry_frac)
reset_prob = 0.04
# setup some symbolic variables and stuff
Xd = T.matrix('Xd_base')
Xc = T.matrix('Xc_base')
Xm = T.matrix('Xm_base')
data_dim = Xtr.shape[1]
prior_sigma = 1.0
Xtr_mean = np.mean(Xtr, axis=0)
##########################
# NETWORK CONFIGURATIONS #
##########################
gn_params = {}
shared_config = [PRIOR_DIM, 1500, 1500]
top_config = [shared_config[-1], data_dim]
gn_params['shared_config'] = shared_config
gn_params['mu_config'] = top_config
gn_params['sigma_config'] = top_config
gn_params['activation'] = relu_actfun
gn_params['init_scale'] = 1.4
gn_params['lam_l2a'] = 0.0
gn_params['vis_drop'] = 0.0
gn_params['hid_drop'] = 0.0
gn_params['bias_noise'] = 0.0
gn_params['input_noise'] = 0.0
# choose some parameters for the continuous inferencer
in_params = {}
shared_config = [data_dim, 1500, 1500]
top_config = [shared_config[-1], PRIOR_DIM]
in_params['shared_config'] = shared_config
in_params['mu_config'] = top_config
in_params['sigma_config'] = top_config
in_params['activation'] = relu_actfun
in_params['init_scale'] = 1.4
in_params['lam_l2a'] = 0.0
in_params['vis_drop'] = 0.0
in_params['hid_drop'] = 0.0
in_params['bias_noise'] = 0.0
in_params['input_noise'] = 0.0
# Initialize the base networks for this OneStageModel
IN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=in_params, shared_param_dicts=None)
GN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=gn_params, shared_param_dicts=None)
# Initialize biases in IN and GN
IN.init_biases(0.2)
GN.init_biases(0.2)
######################################
# LOAD AND RESTART FROM SAVED PARAMS #
######################################
# gn_fname = RESULT_PATH+"pt_osm_params_b110000_GN.pkl"
# in_fname = RESULT_PATH+"pt_osm_params_b110000_IN.pkl"
# IN = load_infnet_from_file(f_name=in_fname, rng=rng, Xd=Xd, \
# new_params=None)
# GN = load_infnet_from_file(f_name=gn_fname, rng=rng, Xd=Xd, \
# new_params=None)
# in_params = IN.params
# gn_params = GN.params
#########################
# INITIALIZE THE GIPAIR #
#########################
osm_params = {}
osm_params['x_type'] = 'bernoulli'
osm_params['xt_transform'] = 'sigmoid'
osm_params['logvar_bound'] = LOGVAR_BOUND
OSM = OneStageModel(rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, \
p_x_given_z=GN, q_z_given_x=IN, \
x_dim=data_dim, z_dim=PRIOR_DIM, params=osm_params)
OSM.set_lam_l2w(1e-5)
safe_mean = (0.9 * Xtr_mean) + 0.05
safe_mean_logit = np.log(safe_mean / (1.0 - safe_mean))
OSM.set_output_bias(safe_mean_logit)
OSM.set_input_bias(-Xtr_mean)
######################
# BASIC VAE TRAINING #
######################
out_file = open(RESULT_PATH+"pt_osm_results.txt", 'wb')
# Set initial learning rate and basic SGD hyper parameters
obs_costs = np.zeros((batch_size,))
costs = [0. for i in range(10)]
learn_rate = 0.002
for i in range(200000):
scale = min(1.0, float(i) / 5000.0)
if ((i > 1) and ((i % 20000) == 0)):
learn_rate = learn_rate * 0.8
if (i < 50000):
momentum = 0.5
elif (i < 10000):
momentum = 0.7
else:
momentum = 0.9
if ((i == 0) or (npr.rand() < reset_prob)):
# sample a fully random batch
batch_idx = npr.randint(low=0,high=tr_samples,size=(batch_size,))
else:
# sample a partially random batch, which retains some portion of
# the worst scoring examples from the previous batch
fresh_idx = npr.randint(low=0,high=tr_samples,size=(batch_size-carry_size,))
batch_idx = np.concatenate((fresh_idx.ravel(), carry_idx.ravel()))
# do a minibatch update of the model, and compute some costs
tr_idx = npr.randint(low=0,high=tr_samples,size=(batch_size,))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xc_batch = 0.0 * Xd_batch
Xm_batch = 0.0 * Xd_batch
# do a minibatch update of the model, and compute some costs
OSM.set_sgd_params(lr_1=(scale*learn_rate), \
mom_1=(scale*momentum), mom_2=0.98)
OSM.set_lam_nll(1.0)
OSM.set_lam_kld(lam_kld_1=scale*lam_kld, lam_kld_2=0.0, lam_kld_c=50.0)
result = OSM.train_joint(Xd_batch, Xc_batch, Xm_batch, batch_reps)
batch_costs = result[4] + result[5]
obs_costs = collect_obs_costs(batch_costs, batch_reps)
carry_idx = batch_idx[np.argsort(-obs_costs)[0:carry_size]]
costs = [(costs[j] + result[j]) for j in range(len(result))]
if ((i % 1000) == 0):
# record and then reset the cost trackers
costs = [(v / 1000.0) for v in costs]
str_1 = "-- batch {0:d} --".format(i)
str_2 = " joint_cost: {0:.4f}".format(costs[0])
str_3 = " nll_cost : {0:.4f}".format(costs[1])
str_4 = " kld_cost : {0:.4f}".format(costs[2])
str_5 = " reg_cost : {0:.4f}".format(costs[3])
costs = [0.0 for v in costs]
# print out some diagnostic information
joint_str = "\n".join([str_1, str_2, str_3, str_4, str_5])
print(joint_str)
out_file.write(joint_str+"\n")
out_file.flush()
if ((i % 2000) == 0):
Xva = row_shuffle(Xva)
model_samps = OSM.sample_from_prior(500)
file_name = RESULT_PATH+"pt_osm_samples_b{0:d}_XG.png".format(i)
utils.visualize_samples(model_samps, file_name, num_rows=20)
file_name = RESULT_PATH+"pt_osm_inf_weights_b{0:d}.png".format(i)
utils.visualize_samples(OSM.inf_weights.get_value(borrow=False).T, \
file_name, num_rows=30)
file_name = RESULT_PATH+"pt_osm_gen_weights_b{0:d}.png".format(i)
utils.visualize_samples(OSM.gen_weights.get_value(borrow=False), \
file_name, num_rows=30)
# compute information about free-energy on validation set
file_name = RESULT_PATH+"pt_osm_free_energy_b{0:d}.png".format(i)
fe_terms = OSM.compute_fe_terms(Xva[0:2500], 20)
fe_mean = np.mean(fe_terms[0]) + np.mean(fe_terms[1])
fe_str = " nll_bound : {0:.4f}".format(fe_mean)
print(fe_str)
out_file.write(fe_str+"\n")
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, \
x_label='Posterior KLd', y_label='Negative Log-likelihood')
# compute information about posterior KLds on validation set
file_name = RESULT_PATH+"pt_osm_post_klds_b{0:d}.png".format(i)
post_klds = OSM.compute_post_klds(Xva[0:2500])
post_dim_klds = np.mean(post_klds, axis=0)
utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
file_name)
if ((i % 5000) == 0):
IN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_b{0:d}_IN.pkl".format(i))
GN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_b{0:d}_GN.pkl".format(i))
IN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_IN.pkl")
GN.save_to_file(f_name=RESULT_PATH+"pt_osm_params_GN.pkl")
return
def pretrain_osm(lam_kld=0.0):
# Initialize a source of randomness
rng = np.random.RandomState(1234)
# Load some data to train/validate/test with
dataset = 'data/mnist.pkl.gz'
datasets = load_udm(dataset, zero_mean=False)
Xtr = datasets[0][0]
Xtr = Xtr.get_value(borrow=False)
Xva = datasets[2][0]
Xva = Xva.get_value(borrow=False)
print("Xtr.shape: {0:s}, Xva.shape: {1:s}".format(str(Xtr.shape),
str(Xva.shape)))
# get and set some basic dataset information
Xtr_mean = np.mean(Xtr, axis=0)
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
batch_size = 100
batch_reps = 5
# setup some symbolic variables and stuff
Xd = T.matrix('Xd_base')
Xc = T.matrix('Xc_base')
Xm = T.matrix('Xm_base')
data_dim = Xtr.shape[1]
prior_sigma = 1.0
##########################
# NETWORK CONFIGURATIONS #
##########################
gn_params = {}
shared_config = [PRIOR_DIM, 1000, 1000]
top_config = [shared_config[-1], data_dim]
gn_params['shared_config'] = shared_config
gn_params['mu_config'] = top_config
gn_params['sigma_config'] = top_config
gn_params['activation'] = relu_actfun
gn_params['init_scale'] = 1.4
gn_params['lam_l2a'] = 0.0
gn_params['vis_drop'] = 0.0
gn_params['hid_drop'] = 0.0
gn_params['bias_noise'] = 0.0
gn_params['input_noise'] = 0.0
# choose some parameters for the continuous inferencer
in_params = {}
shared_config = [data_dim, 1000, 1000]
top_config = [shared_config[-1], PRIOR_DIM]
in_params['shared_config'] = shared_config
in_params['mu_config'] = top_config
in_params['sigma_config'] = top_config
in_params['activation'] = relu_actfun
in_params['init_scale'] = 1.4
in_params['lam_l2a'] = 0.0
in_params['vis_drop'] = 0.0
in_params['hid_drop'] = 0.0
in_params['bias_noise'] = 0.0
in_params['input_noise'] = 0.0
# Initialize the base networks for this OneStageModel
IN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=in_params, shared_param_dicts=None)
GN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=gn_params, shared_param_dicts=None)
# Initialize biases in IN and GN
IN.init_biases(0.2)
GN.init_biases(0.2)
#########################
# INITIALIZE THE GIPAIR #
#########################
osm_params = {}
osm_params['x_type'] = 'bernoulli'
osm_params['xt_transform'] = 'sigmoid'
osm_params['logvar_bound'] = LOGVAR_BOUND
OSM = OneStageModel(rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, \
p_x_given_z=GN, q_z_given_x=IN, \
x_dim=data_dim, z_dim=PRIOR_DIM, params=osm_params)
OSM.set_lam_l2w(1e-5)
safe_mean = (0.9 * Xtr_mean) + 0.05
safe_mean_logit = np.log(safe_mean / (1.0 - safe_mean))
OSM.set_output_bias(safe_mean_logit)
OSM.set_input_bias(-Xtr_mean)
######################
# BASIC VAE TRAINING #
######################
out_file = open(RESULT_PATH + "pt_osm_results.txt", 'wb')
# Set initial learning rate and basic SGD hyper parameters
obs_costs = np.zeros((batch_size, ))
costs = [0. for i in range(10)]
learn_rate = 0.0005
for i in range(150000):
scale = min(1.0, float(i) / 10000.0)
if ((i > 1) and ((i % 20000) == 0)):
learn_rate = learn_rate * 0.9
# do a minibatch update of the model, and compute some costs
tr_idx = npr.randint(low=0, high=tr_samples, size=(batch_size, ))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xc_batch = 0.0 * Xd_batch
Xm_batch = 0.0 * Xd_batch
# do a minibatch update of the model, and compute some costs
OSM.set_sgd_params(lr_1=(scale * learn_rate), mom_1=0.5, mom_2=0.98)
OSM.set_lam_nll(1.0)
OSM.set_lam_kld(lam_kld_1=(1.0 + (scale * (lam_kld - 1.0))),
lam_kld_2=0.0)
result = OSM.train_joint(Xd_batch, Xc_batch, Xm_batch, batch_reps)
costs = [(costs[j] + result[j]) for j in range(len(result))]
if ((i % 1000) == 0):
# record and then reset the cost trackers
costs = [(v / 1000.0) for v in costs]
str_1 = "-- batch {0:d} --".format(i)
str_2 = " joint_cost: {0:.4f}".format(costs[0])
str_3 = " nll_cost : {0:.4f}".format(costs[1])
str_4 = " kld_cost : {0:.4f}".format(costs[2])
str_5 = " reg_cost : {0:.4f}".format(costs[3])
costs = [0.0 for v in costs]
# print out some diagnostic information
joint_str = "\n".join([str_1, str_2, str_3, str_4, str_5])
print(joint_str)
out_file.write(joint_str + "\n")
out_file.flush()
if ((i % 2000) == 0):
Xva = row_shuffle(Xva)
model_samps = OSM.sample_from_prior(500)
file_name = RESULT_PATH + "pt_osm_samples_b{0:d}_XG.png".format(i)
utils.visualize_samples(model_samps, file_name, num_rows=20)
# compute information about free-energy on validation set
file_name = RESULT_PATH + "pt_osm_free_energy_b{0:d}.png".format(i)
fe_terms = OSM.compute_fe_terms(Xva[0:2500], 20)
fe_mean = np.mean(fe_terms[0]) + np.mean(fe_terms[1])
fe_str = " nll_bound : {0:.4f}".format(fe_mean)
print(fe_str)
out_file.write(fe_str + "\n")
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, \
x_label='Posterior KLd', y_label='Negative Log-likelihood')
# compute information about posterior KLds on validation set
file_name = RESULT_PATH + "pt_osm_post_klds_b{0:d}.png".format(i)
post_klds = OSM.compute_post_klds(Xva[0:2500])
post_dim_klds = np.mean(post_klds, axis=0)
utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
file_name)
if ((i % 5000) == 0):
IN.save_to_file(f_name=RESULT_PATH +
"pt_osm_params_b{0:d}_IN.pkl".format(i))
GN.save_to_file(f_name=RESULT_PATH +
"pt_osm_params_b{0:d}_GN.pkl".format(i))
IN.save_to_file(f_name=RESULT_PATH + "pt_osm_params_IN.pkl")
GN.save_to_file(f_name=RESULT_PATH + "pt_osm_params_GN.pkl")
return
def pretrain_osm(lam_kld=0.0):
# Initialize a source of randomness
rng = np.random.RandomState(1234)
# Load some data to train/validate/test with
data_file = 'data/tfd_data_48x48.pkl'
dataset = load_tfd(tfd_pkl_name=data_file,
which_set='unlabeled',
fold='all')
Xtr_unlabeled = dataset[0]
dataset = load_tfd(tfd_pkl_name=data_file, which_set='train', fold='all')
Xtr_train = dataset[0]
Xtr = np.vstack([Xtr_unlabeled, Xtr_train])
dataset = load_tfd(tfd_pkl_name=data_file, which_set='valid', fold='all')
Xva = dataset[0]
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
batch_size = 400
batch_reps = 6
carry_frac = 0.25
carry_size = int(batch_size * carry_frac)
reset_prob = 0.04
# setup some symbolic variables and stuff
Xd = T.matrix('Xd_base')
Xc = T.matrix('Xc_base')
Xm = T.matrix('Xm_base')
data_dim = Xtr.shape[1]
prior_sigma = 1.0
Xtr_mean = np.mean(Xtr, axis=0)
##########################
# NETWORK CONFIGURATIONS #
##########################
gn_params = {}
shared_config = [PRIOR_DIM, 1500, 1500]
top_config = [shared_config[-1], data_dim]
gn_params['shared_config'] = shared_config
gn_params['mu_config'] = top_config
gn_params['sigma_config'] = top_config
gn_params['activation'] = relu_actfun
gn_params['init_scale'] = 1.4
gn_params['lam_l2a'] = 0.0
gn_params['vis_drop'] = 0.0
gn_params['hid_drop'] = 0.0
gn_params['bias_noise'] = 0.0
gn_params['input_noise'] = 0.0
# choose some parameters for the continuous inferencer
in_params = {}
shared_config = [data_dim, 1500, 1500]
top_config = [shared_config[-1], PRIOR_DIM]
in_params['shared_config'] = shared_config
in_params['mu_config'] = top_config
in_params['sigma_config'] = top_config
in_params['activation'] = relu_actfun
in_params['init_scale'] = 1.4
in_params['lam_l2a'] = 0.0
in_params['vis_drop'] = 0.0
in_params['hid_drop'] = 0.0
in_params['bias_noise'] = 0.0
in_params['input_noise'] = 0.0
# Initialize the base networks for this OneStageModel
IN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=in_params, shared_param_dicts=None)
GN = InfNet(rng=rng, Xd=Xd, prior_sigma=prior_sigma, \
params=gn_params, shared_param_dicts=None)
# Initialize biases in IN and GN
IN.init_biases(0.2)
GN.init_biases(0.2)
######################################
# LOAD AND RESTART FROM SAVED PARAMS #
######################################
# gn_fname = RESULT_PATH+"pt_osm_params_b110000_GN.pkl"
# in_fname = RESULT_PATH+"pt_osm_params_b110000_IN.pkl"
# IN = load_infnet_from_file(f_name=in_fname, rng=rng, Xd=Xd, \
# new_params=None)
# GN = load_infnet_from_file(f_name=gn_fname, rng=rng, Xd=Xd, \
# new_params=None)
# in_params = IN.params
# gn_params = GN.params
#########################
# INITIALIZE THE GIPAIR #
#########################
osm_params = {}
osm_params['x_type'] = 'bernoulli'
osm_params['xt_transform'] = 'sigmoid'
osm_params['logvar_bound'] = LOGVAR_BOUND
OSM = OneStageModel(rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, \
p_x_given_z=GN, q_z_given_x=IN, \
x_dim=data_dim, z_dim=PRIOR_DIM, params=osm_params)
OSM.set_lam_l2w(1e-5)
safe_mean = (0.9 * Xtr_mean) + 0.05
safe_mean_logit = np.log(safe_mean / (1.0 - safe_mean))
OSM.set_output_bias(safe_mean_logit)
OSM.set_input_bias(-Xtr_mean)
######################
# BASIC VAE TRAINING #
######################
out_file = open(RESULT_PATH + "pt_osm_results.txt", 'wb')
# Set initial learning rate and basic SGD hyper parameters
obs_costs = np.zeros((batch_size, ))
costs = [0. for i in range(10)]
learn_rate = 0.002
for i in range(200000):
scale = min(1.0, float(i) / 5000.0)
if ((i > 1) and ((i % 20000) == 0)):
learn_rate = learn_rate * 0.8
if (i < 50000):
momentum = 0.5
elif (i < 10000):
momentum = 0.7
else:
momentum = 0.9
if ((i == 0) or (npr.rand() < reset_prob)):
# sample a fully random batch
batch_idx = npr.randint(low=0,
high=tr_samples,
size=(batch_size, ))
else:
# sample a partially random batch, which retains some portion of
# the worst scoring examples from the previous batch
fresh_idx = npr.randint(low=0,
high=tr_samples,
size=(batch_size - carry_size, ))
batch_idx = np.concatenate((fresh_idx.ravel(), carry_idx.ravel()))
# do a minibatch update of the model, and compute some costs
tr_idx = npr.randint(low=0, high=tr_samples, size=(batch_size, ))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xc_batch = 0.0 * Xd_batch
Xm_batch = 0.0 * Xd_batch
# do a minibatch update of the model, and compute some costs
OSM.set_sgd_params(lr_1=(scale*learn_rate), \
mom_1=(scale*momentum), mom_2=0.98)
OSM.set_lam_nll(1.0)
OSM.set_lam_kld(lam_kld_1=scale * lam_kld,
lam_kld_2=0.0,
lam_kld_c=50.0)
result = OSM.train_joint(Xd_batch, Xc_batch, Xm_batch, batch_reps)
batch_costs = result[4] + result[5]
obs_costs = collect_obs_costs(batch_costs, batch_reps)
carry_idx = batch_idx[np.argsort(-obs_costs)[0:carry_size]]
costs = [(costs[j] + result[j]) for j in range(len(result))]
if ((i % 1000) == 0):
# record and then reset the cost trackers
costs = [(v / 1000.0) for v in costs]
str_1 = "-- batch {0:d} --".format(i)
str_2 = " joint_cost: {0:.4f}".format(costs[0])
str_3 = " nll_cost : {0:.4f}".format(costs[1])
str_4 = " kld_cost : {0:.4f}".format(costs[2])
str_5 = " reg_cost : {0:.4f}".format(costs[3])
costs = [0.0 for v in costs]
# print out some diagnostic information
joint_str = "\n".join([str_1, str_2, str_3, str_4, str_5])
print(joint_str)
out_file.write(joint_str + "\n")
out_file.flush()
if ((i % 2000) == 0):
Xva = row_shuffle(Xva)
model_samps = OSM.sample_from_prior(500)
file_name = RESULT_PATH + "pt_osm_samples_b{0:d}_XG.png".format(i)
utils.visualize_samples(model_samps, file_name, num_rows=20)
file_name = RESULT_PATH + "pt_osm_inf_weights_b{0:d}.png".format(i)
utils.visualize_samples(OSM.inf_weights.get_value(borrow=False).T, \
file_name, num_rows=30)
file_name = RESULT_PATH + "pt_osm_gen_weights_b{0:d}.png".format(i)
utils.visualize_samples(OSM.gen_weights.get_value(borrow=False), \
file_name, num_rows=30)
# compute information about free-energy on validation set
file_name = RESULT_PATH + "pt_osm_free_energy_b{0:d}.png".format(i)
fe_terms = OSM.compute_fe_terms(Xva[0:2500], 20)
fe_mean = np.mean(fe_terms[0]) + np.mean(fe_terms[1])
fe_str = " nll_bound : {0:.4f}".format(fe_mean)
print(fe_str)
out_file.write(fe_str + "\n")
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, \
x_label='Posterior KLd', y_label='Negative Log-likelihood')
# compute information about posterior KLds on validation set
file_name = RESULT_PATH + "pt_osm_post_klds_b{0:d}.png".format(i)
post_klds = OSM.compute_post_klds(Xva[0:2500])
post_dim_klds = np.mean(post_klds, axis=0)
utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
file_name)
if ((i % 5000) == 0):
IN.save_to_file(f_name=RESULT_PATH +
"pt_osm_params_b{0:d}_IN.pkl".format(i))
GN.save_to_file(f_name=RESULT_PATH +
"pt_osm_params_b{0:d}_GN.pkl".format(i))
IN.save_to_file(f_name=RESULT_PATH + "pt_osm_params_IN.pkl")
GN.save_to_file(f_name=RESULT_PATH + "pt_osm_params_GN.pkl")
return
def test_gip_sigma_scale_tfd():
from LogPDFs import cross_validate_sigma
# Simple test code, to check that everything is basically functional.
print("TESTING...")
# Initialize a source of randomness
rng = np.random.RandomState(12345)
# Load some data to train/validate/test with
data_file = "data/tfd_data_48x48.pkl"
dataset = load_tfd(tfd_pkl_name=data_file, which_set="unlabeled", fold="all")
Xtr_unlabeled = dataset[0]
dataset = load_tfd(tfd_pkl_name=data_file, which_set="train", fold="all")
Xtr_train = dataset[0]
Xtr = np.vstack([Xtr_unlabeled, Xtr_train])
dataset = load_tfd(tfd_pkl_name=data_file, which_set="test", fold="all")
Xva = dataset[0]
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
print("Xtr.shape: {0:s}, Xva.shape: {1:s}".format(str(Xtr.shape), str(Xva.shape)))
# get and set some basic dataset information
tr_samples = Xtr.shape[0]
data_dim = Xtr.shape[1]
batch_size = 100
# Symbolic inputs
Xd = T.matrix(name="Xd")
Xc = T.matrix(name="Xc")
Xm = T.matrix(name="Xm")
Xt = T.matrix(name="Xt")
# Load inferencer and generator from saved parameters
gn_fname = "TFD_WALKOUT_TEST_KLD/pt_walk_params_b25000_GN.pkl"
in_fname = "TFD_WALKOUT_TEST_KLD/pt_walk_params_b25000_IN.pkl"
IN = load_infnet_from_file(f_name=in_fname, rng=rng, Xd=Xd)
GN = load_infnet_from_file(f_name=gn_fname, rng=rng, Xd=Xd)
x_dim = IN.shared_layers[0].in_dim
z_dim = IN.mu_layers[-1].out_dim
# construct a GIPair with the loaded InfNet and GenNet
osm_params = {}
osm_params["x_type"] = "gaussian"
osm_params["xt_transform"] = "sigmoid"
osm_params["logvar_bound"] = LOGVAR_BOUND
OSM = OneStageModel(
rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, p_x_given_z=GN, q_z_given_x=IN, x_dim=x_dim, z_dim=z_dim, params=osm_params
)
# # compute variational likelihood bound and its sub-components
Xva = row_shuffle(Xva)
Xb = Xva[0:5000]
# file_name = "A_TFD_POST_KLDS.png"
# post_klds = OSM.compute_post_klds(Xb)
# post_dim_klds = np.mean(post_klds, axis=0)
# utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
# file_name)
# compute information about free-energy on validation set
file_name = "A_TFD_KLD_FREE_ENERGY.png"
fe_terms = OSM.compute_fe_terms(Xb, 20)
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, x_label="Posterior KLd", y_label="Negative Log-likelihood")
# bound_results = OSM.compute_ll_bound(Xva)
# ll_bounds = bound_results[0]
# post_klds = bound_results[1]
# log_likelihoods = bound_results[2]
# max_lls = bound_results[3]
# print("mean ll bound: {0:.4f}".format(np.mean(ll_bounds)))
# print("mean posterior KLd: {0:.4f}".format(np.mean(post_klds)))
# print("mean log-likelihood: {0:.4f}".format(np.mean(log_likelihoods)))
# print("mean max log-likelihood: {0:.4f}".format(np.mean(max_lls)))
# print("min ll bound: {0:.4f}".format(np.min(ll_bounds)))
# print("max posterior KLd: {0:.4f}".format(np.max(post_klds)))
# print("min log-likelihood: {0:.4f}".format(np.min(log_likelihoods)))
# print("min max log-likelihood: {0:.4f}".format(np.min(max_lls)))
# # compute some information about the approximate posteriors
# post_stats = OSM.compute_post_stats(Xva, 0.0*Xva, 0.0*Xva)
# all_post_klds = np.sort(post_stats[0].ravel()) # post KLds for each obs and dim
# obs_post_klds = np.sort(post_stats[1]) # summed post KLds for each obs
# post_dim_klds = post_stats[2] # average post KLds for each post dim
# post_dim_vars = post_stats[3] # average squared mean for each post dim
# utils.plot_line(np.arange(all_post_klds.shape[0]), all_post_klds, "AAA_ALL_POST_KLDS.png")
# utils.plot_line(np.arange(obs_post_klds.shape[0]), obs_post_klds, "AAA_OBS_POST_KLDS.png")
# utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, "AAA_POST_DIM_KLDS.png")
# utils.plot_stem(np.arange(post_dim_vars.shape[0]), post_dim_vars, "AAA_POST_DIM_VARS.png")
# draw many samples from the GIP
for i in range(5):
tr_idx = npr.randint(low=0, high=tr_samples, size=(100,))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xs = []
for row in range(3):
Xs.append([])
for col in range(3):
sample_lists = OSM.sample_from_chain(Xd_batch[0:10, :], loop_iters=100, sigma_scale=1.0)
Xs[row].append(group_chains(sample_lists["data samples"]))
Xs, block_im_dim = block_video(Xs, (48, 48), (3, 3))
to_video(Xs, block_im_dim, "A_TFD_KLD_CHAIN_VIDEO_{0:d}.avi".format(i), frame_rate=10)
# sample_lists = GIP.sample_from_chain(Xd_batch[0,:].reshape((1,data_dim)), loop_iters=300, \
# sigma_scale=1.0)
# Xs = np.vstack(sample_lists["data samples"])
# file_name = "TFD_TEST_{0:d}.png".format(i)
# utils.visualize_samples(Xs, file_name, num_rows=15)
file_name = "A_TFD_KLD_PRIOR_SAMPLE.png"
Xs = OSM.sample_from_prior(20 * 20)
utils.visualize_samples(Xs, file_name, num_rows=20)
# test Parzen density estimator built from prior samples
# Xs = OSM.sample_from_prior(10000)
# [best_sigma, best_ll, best_lls] = \
# cross_validate_sigma(Xs, Xva, [0.09, 0.095, 0.1, 0.105, 0.11], 10)
# sort_idx = np.argsort(best_lls)
# sort_idx = sort_idx[0:400]
# utils.plot_line(np.arange(sort_idx.shape[0]), best_lls[sort_idx], "A_TFD_BEST_LLS_1.png")
# utils.visualize_samples(Xva[sort_idx], "A_TFD_BAD_FACES_1.png", num_rows=20)
return
def test_gip_sigma_scale_mnist():
from LogPDFs import cross_validate_sigma
# Simple test code, to check that everything is basically functional.
print("TESTING...")
# Initialize a source of randomness
rng = np.random.RandomState(12345)
# Load some data to train/validate/test with
dataset = 'data/mnist.pkl.gz'
datasets = load_udm(dataset, zero_mean=False)
Xtr = datasets[0][0]
Xtr = Xtr.get_value(borrow=False)
Xva = datasets[2][0]
Xva = Xva.get_value(borrow=False)
print("Xtr.shape: {0:s}, Xva.shape: {1:s}".format(str(Xtr.shape),str(Xva.shape)))
# get and set some basic dataset information
tr_samples = Xtr.shape[0]
batch_size = 100
Xtr_mean = np.mean(Xtr, axis=0, keepdims=True)
Xtr_mean = (0.0 * Xtr_mean) + np.mean(Xtr)
Xc_mean = np.repeat(Xtr_mean, batch_size, axis=0).astype(theano.config.floatX)
# Symbolic inputs
Xd = T.matrix(name='Xd')
Xc = T.matrix(name='Xc')
Xm = T.matrix(name='Xm')
Xt = T.matrix(name='Xt')
# Load inferencer and generator from saved parameters
gn_fname = "MNIST_WALKOUT_TEST_MAX_KLD/pt_walk_params_b70000_GN.pkl"
in_fname = "MNIST_WALKOUT_TEST_MAX_KLD/pt_walk_params_b70000_IN.pkl"
IN = load_infnet_from_file(f_name=in_fname, rng=rng, Xd=Xd)
GN = load_infnet_from_file(f_name=gn_fname, rng=rng, Xd=Xd)
x_dim = IN.shared_layers[0].in_dim
z_dim = IN.mu_layers[-1].out_dim
# construct a GIPair with the loaded InfNet and GenNet
osm_params = {}
osm_params['x_type'] = 'gaussian'
osm_params['xt_transform'] = 'sigmoid'
osm_params['logvar_bound'] = LOGVAR_BOUND
OSM = OneStageModel(rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, \
p_x_given_z=GN, q_z_given_x=IN, \
x_dim=x_dim, z_dim=z_dim, params=osm_params)
# compute variational likelihood bound and its sub-components
Xva = row_shuffle(Xva)
Xb = Xva[0:5000]
file_name = "A_MNIST_POST_KLDS.png"
post_klds = OSM.compute_post_klds(Xb)
post_dim_klds = np.mean(post_klds, axis=0)
utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
file_name)
# compute information about free-energy on validation set
file_name = "A_MNIST_FREE_ENERGY.png"
fe_terms = OSM.compute_fe_terms(Xb, 20)
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, \
x_label='Posterior KLd', y_label='Negative Log-likelihood')
# bound_results = OSM.compute_ll_bound(Xva)
# ll_bounds = bound_results[0]
# post_klds = bound_results[1]
# log_likelihoods = bound_results[2]
# max_lls = bound_results[3]
# print("mean ll bound: {0:.4f}".format(np.mean(ll_bounds)))
# print("mean posterior KLd: {0:.4f}".format(np.mean(post_klds)))
# print("mean log-likelihood: {0:.4f}".format(np.mean(log_likelihoods)))
# print("mean max log-likelihood: {0:.4f}".format(np.mean(max_lls)))
# print("min ll bound: {0:.4f}".format(np.min(ll_bounds)))
# print("max posterior KLd: {0:.4f}".format(np.max(post_klds)))
# print("min log-likelihood: {0:.4f}".format(np.min(log_likelihoods)))
# print("min max log-likelihood: {0:.4f}".format(np.min(max_lls)))
# # compute some information about the approximate posteriors
# post_stats = OSM.compute_post_stats(Xva, 0.0*Xva, 0.0*Xva)
# all_post_klds = np.sort(post_stats[0].ravel()) # post KLds for each obs and dim
# obs_post_klds = np.sort(post_stats[1]) # summed post KLds for each obs
# post_dim_klds = post_stats[2] # average post KLds for each post dim
# post_dim_vars = post_stats[3] # average squared mean for each post dim
# utils.plot_line(np.arange(all_post_klds.shape[0]), all_post_klds, "AAA_ALL_POST_KLDS.png")
# utils.plot_line(np.arange(obs_post_klds.shape[0]), obs_post_klds, "AAA_OBS_POST_KLDS.png")
# utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, "AAA_POST_DIM_KLDS.png")
# utils.plot_stem(np.arange(post_dim_vars.shape[0]), post_dim_vars, "AAA_POST_DIM_VARS.png")
# draw many samples from the GIP
for i in range(5):
tr_idx = npr.randint(low=0,high=tr_samples,size=(100,))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xs = []
for row in range(3):
Xs.append([])
for col in range(3):
sample_lists = OSM.sample_from_chain(Xd_batch[0:10,:], loop_iters=100, \
sigma_scale=1.0)
Xs[row].append(group_chains(sample_lists['data samples']))
Xs, block_im_dim = block_video(Xs, (28,28), (3,3))
to_video(Xs, block_im_dim, "A_MNIST_KLD_CHAIN_VIDEO_{0:d}.avi".format(i), frame_rate=10)
#sample_lists = GIP.sample_from_chain(Xd_batch[0,:].reshape((1,data_dim)), loop_iters=300, \
# sigma_scale=1.0)
#Xs = np.vstack(sample_lists["data samples"])
#file_name = "TFD_TEST_{0:d}.png".format(i)
#utils.visualize_samples(Xs, file_name, num_rows=15)
file_name = "A_MNIST_KLD_PRIOR_SAMPLE.png"
Xs = OSM.sample_from_prior(20*20)
utils.visualize_samples(Xs, file_name, num_rows=20)
# # test Parzen density estimator built from prior samples
# Xs = OSM.sample_from_prior(10000)
# [best_sigma, best_ll, best_lls] = \
# cross_validate_sigma(Xs, Xva, [0.12, 0.14, 0.15, 0.16, 0.18], 20)
# sort_idx = np.argsort(best_lls)
# sort_idx = sort_idx[0:400]
# utils.plot_line(np.arange(sort_idx.shape[0]), best_lls[sort_idx], "A_MNIST_BEST_LLS_1.png")
# utils.visualize_samples(Xva[sort_idx], "A_MNIST_BAD_DIGITS_1.png", num_rows=20)
# ##########
# # AGAIN! #
# ##########
# Xs = OSM.sample_from_prior(10000)
# tr_idx = npr.randint(low=0,high=tr_samples,size=(5000,))
# Xva = Xtr.take(tr_idx, axis=0)
# [best_sigma, best_ll, best_lls] = \
# cross_validate_sigma(Xs, Xva, [0.12, 0.14, 0.15, 0.16, 0.18], 20)
# sort_idx = np.argsort(best_lls)
# sort_idx = sort_idx[0:400]
# utils.plot_line(np.arange(sort_idx.shape[0]), best_lls[sort_idx], "A_MNIST_BEST_LLS_2.png")
# utils.visualize_samples(Xva[sort_idx], "A_MNIST_BAD_DIGITS_2.png", num_rows=20)
return
def test_one_stage_model():
##########################
# Get some training data #
##########################
rng = np.random.RandomState(1234)
Xtr, Xva, Xte = load_binarized_mnist(data_path='./data/')
Xtr = np.vstack((Xtr, Xva))
Xva = Xte
#del Xte
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
batch_size = 128
batch_reps = 1
###############################################
# Setup some parameters for the OneStageModel #
###############################################
x_dim = Xtr.shape[1]
z_dim = 64
x_type = 'bernoulli'
xin_sym = T.matrix('xin_sym')
###############
# p_x_given_z #
###############
params = {}
shared_config = \
[ {'layer_type': 'fc',
'in_chans': z_dim,
'out_chans': 256,
'activation': relu_actfun,
'apply_bn': True}, \
{'layer_type': 'fc',
'in_chans': 256,
'out_chans': 7*7*128,
'activation': relu_actfun,
'apply_bn': True,
'shape_func_out': lambda x: T.reshape(x, (-1, 128, 7, 7))}, \
{'layer_type': 'conv',
'in_chans': 128, # in shape: (batch, 128, 7, 7)
'out_chans': 64, # out shape: (batch, 64, 14, 14)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'half',
'apply_bn': True} ]
output_config = \
[ {'layer_type': 'conv',
'in_chans': 64, # in shape: (batch, 64, 14, 14)
'out_chans': 1, # out shape: (batch, 1, 28, 28)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'half',
'apply_bn': False,
'shape_func_out': lambda x: T.flatten(x, 2)}, \
{'layer_type': 'conv',
'in_chans': 64,
'out_chans': 1,
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'half',
'apply_bn': False,
'shape_func_out': lambda x: T.flatten(x, 2)} ]
params['shared_config'] = shared_config
params['output_config'] = output_config
params['init_scale'] = 1.0
params['build_theano_funcs'] = False
p_x_given_z = HydraNet(rng=rng, Xd=xin_sym, \
params=params, shared_param_dicts=None)
p_x_given_z.init_biases(0.0)
###############
# q_z_given_x #
###############
params = {}
shared_config = \
[ {'layer_type': 'conv',
'in_chans': 1, # in shape: (batch, 784)
'out_chans': 64, # out shape: (batch, 64, 14, 14)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'double',
'apply_bn': True,
'shape_func_in': lambda x: T.reshape(x, (-1, 1, 28, 28))}, \
{'layer_type': 'conv',
'in_chans': 64, # in shape: (batch, 64, 14, 14)
'out_chans': 128, # out shape: (batch, 128, 7, 7)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'double',
'apply_bn': True,
'shape_func_out': lambda x: T.flatten(x, 2)}, \
{'layer_type': 'fc',
'in_chans': 128*7*7,
'out_chans': 256,
'activation': relu_actfun,
'apply_bn': True} ]
output_config = \
[ {'layer_type': 'fc',
'in_chans': 256,
'out_chans': z_dim,
'activation': relu_actfun,
'apply_bn': False}, \
{'layer_type': 'fc',
'in_chans': 256,
'out_chans': z_dim,
'activation': relu_actfun,
'apply_bn': False} ]
params['shared_config'] = shared_config
params['output_config'] = output_config
params['init_scale'] = 1.0
params['build_theano_funcs'] = False
q_z_given_x = HydraNet(rng=rng, Xd=xin_sym, \
params=params, shared_param_dicts=None)
q_z_given_x.init_biases(0.0)
##############################################################
# Define parameters for the TwoStageModel, and initialize it #
##############################################################
print("Building the OneStageModel...")
osm_params = {}
osm_params['x_type'] = x_type
osm_params['obs_transform'] = 'sigmoid'
OSM = OneStageModel(rng=rng, x_in=xin_sym,
x_dim=x_dim, z_dim=z_dim,
p_x_given_z=p_x_given_z,
q_z_given_x=q_z_given_x,
params=osm_params)
################################################################
# Apply some updates, to check that they aren't totally broken #
################################################################
log_name = "{}_RESULTS.txt".format("OSM_TEST")
out_file = open(log_name, 'wb')
costs = [0. for i in range(10)]
learn_rate = 0.0005
momentum = 0.9
batch_idx = np.arange(batch_size) + tr_samples
for i in range(500000):
scale = min(0.5, ((i+1) / 5000.0))
if (((i + 1) % 10000) == 0):
learn_rate = learn_rate * 0.95
# get the indices of training samples for this batch update
batch_idx += batch_size
if (np.max(batch_idx) >= tr_samples):
# we finished an "epoch", so we rejumble the training set
Xtr = row_shuffle(Xtr)
batch_idx = np.arange(batch_size)
Xb = to_fX( Xtr.take(batch_idx, axis=0) )
#Xb = binarize_data(Xtr.take(batch_idx, axis=0))
# set sgd and objective function hyperparams for this update
OSM.set_sgd_params(lr=scale*learn_rate, \
mom_1=(scale*momentum), mom_2=0.98)
OSM.set_lam_nll(lam_nll=1.0)
OSM.set_lam_kld(lam_kld=1.0)
OSM.set_lam_l2w(1e-5)
# perform a minibatch update and record the cost for this batch
result = OSM.train_joint(Xb, batch_reps)
costs = [(costs[j] + result[j]) for j in range(len(result))]
if ((i % 500) == 0):
costs = [(v / 500.0) for v in costs]
str1 = "-- batch {0:d} --".format(i)
str2 = " joint_cost: {0:.4f}".format(costs[0])
str3 = " nll_cost : {0:.4f}".format(costs[1])
str4 = " kld_cost : {0:.4f}".format(costs[2])
str5 = " reg_cost : {0:.4f}".format(costs[3])
joint_str = "\n".join([str1, str2, str3, str4, str5])
print(joint_str)
out_file.write(joint_str+"\n")
out_file.flush()
costs = [0.0 for v in costs]
if (((i % 5000) == 0) or ((i < 10000) and ((i % 1000) == 0))):
# draw some independent random samples from the model
samp_count = 300
model_samps = OSM.sample_from_prior(samp_count)
file_name = "OSM_SAMPLES_b{0:d}.png".format(i)
utils.visualize_samples(model_samps, file_name, num_rows=15)
# compute free energy estimate for validation samples
Xva = row_shuffle(Xva)
fe_terms = OSM.compute_fe_terms(Xva[0:5000], 20)
fe_mean = np.mean(fe_terms[0]) + np.mean(fe_terms[1])
out_str = " nll_bound : {0:.4f}".format(fe_mean)
print(out_str)
out_file.write(out_str+"\n")
out_file.flush()
return
def test_gip_sigma_scale_tfd():
from LogPDFs import cross_validate_sigma
# Simple test code, to check that everything is basically functional.
print("TESTING...")
# Initialize a source of randomness
rng = np.random.RandomState(12345)
# Load some data to train/validate/test with
data_file = 'data/tfd_data_48x48.pkl'
dataset = load_tfd(tfd_pkl_name=data_file,
which_set='unlabeled',
fold='all')
Xtr_unlabeled = dataset[0]
dataset = load_tfd(tfd_pkl_name=data_file, which_set='train', fold='all')
Xtr_train = dataset[0]
Xtr = np.vstack([Xtr_unlabeled, Xtr_train])
dataset = load_tfd(tfd_pkl_name=data_file, which_set='test', fold='all')
Xva = dataset[0]
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
print("Xtr.shape: {0:s}, Xva.shape: {1:s}".format(str(Xtr.shape),
str(Xva.shape)))
# get and set some basic dataset information
tr_samples = Xtr.shape[0]
data_dim = Xtr.shape[1]
batch_size = 100
# Symbolic inputs
Xd = T.matrix(name='Xd')
Xc = T.matrix(name='Xc')
Xm = T.matrix(name='Xm')
Xt = T.matrix(name='Xt')
# Load inferencer and generator from saved parameters
gn_fname = "TFD_WALKOUT_TEST_KLD/pt_walk_params_b25000_GN.pkl"
in_fname = "TFD_WALKOUT_TEST_KLD/pt_walk_params_b25000_IN.pkl"
IN = load_infnet_from_file(f_name=in_fname, rng=rng, Xd=Xd)
GN = load_infnet_from_file(f_name=gn_fname, rng=rng, Xd=Xd)
x_dim = IN.shared_layers[0].in_dim
z_dim = IN.mu_layers[-1].out_dim
# construct a GIPair with the loaded InfNet and GenNet
osm_params = {}
osm_params['x_type'] = 'gaussian'
osm_params['xt_transform'] = 'sigmoid'
osm_params['logvar_bound'] = LOGVAR_BOUND
OSM = OneStageModel(rng=rng, Xd=Xd, Xc=Xc, Xm=Xm, \
p_x_given_z=GN, q_z_given_x=IN, \
x_dim=x_dim, z_dim=z_dim, params=osm_params)
# # compute variational likelihood bound and its sub-components
Xva = row_shuffle(Xva)
Xb = Xva[0:5000]
# file_name = "A_TFD_POST_KLDS.png"
# post_klds = OSM.compute_post_klds(Xb)
# post_dim_klds = np.mean(post_klds, axis=0)
# utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, \
# file_name)
# compute information about free-energy on validation set
file_name = "A_TFD_KLD_FREE_ENERGY.png"
fe_terms = OSM.compute_fe_terms(Xb, 20)
utils.plot_scatter(fe_terms[1], fe_terms[0], file_name, \
x_label='Posterior KLd', y_label='Negative Log-likelihood')
# bound_results = OSM.compute_ll_bound(Xva)
# ll_bounds = bound_results[0]
# post_klds = bound_results[1]
# log_likelihoods = bound_results[2]
# max_lls = bound_results[3]
# print("mean ll bound: {0:.4f}".format(np.mean(ll_bounds)))
# print("mean posterior KLd: {0:.4f}".format(np.mean(post_klds)))
# print("mean log-likelihood: {0:.4f}".format(np.mean(log_likelihoods)))
# print("mean max log-likelihood: {0:.4f}".format(np.mean(max_lls)))
# print("min ll bound: {0:.4f}".format(np.min(ll_bounds)))
# print("max posterior KLd: {0:.4f}".format(np.max(post_klds)))
# print("min log-likelihood: {0:.4f}".format(np.min(log_likelihoods)))
# print("min max log-likelihood: {0:.4f}".format(np.min(max_lls)))
# # compute some information about the approximate posteriors
# post_stats = OSM.compute_post_stats(Xva, 0.0*Xva, 0.0*Xva)
# all_post_klds = np.sort(post_stats[0].ravel()) # post KLds for each obs and dim
# obs_post_klds = np.sort(post_stats[1]) # summed post KLds for each obs
# post_dim_klds = post_stats[2] # average post KLds for each post dim
# post_dim_vars = post_stats[3] # average squared mean for each post dim
# utils.plot_line(np.arange(all_post_klds.shape[0]), all_post_klds, "AAA_ALL_POST_KLDS.png")
# utils.plot_line(np.arange(obs_post_klds.shape[0]), obs_post_klds, "AAA_OBS_POST_KLDS.png")
# utils.plot_stem(np.arange(post_dim_klds.shape[0]), post_dim_klds, "AAA_POST_DIM_KLDS.png")
# utils.plot_stem(np.arange(post_dim_vars.shape[0]), post_dim_vars, "AAA_POST_DIM_VARS.png")
# draw many samples from the GIP
for i in range(5):
tr_idx = npr.randint(low=0, high=tr_samples, size=(100, ))
Xd_batch = Xtr.take(tr_idx, axis=0)
Xs = []
for row in range(3):
Xs.append([])
for col in range(3):
sample_lists = OSM.sample_from_chain(Xd_batch[0:10,:], loop_iters=100, \
sigma_scale=1.0)
Xs[row].append(group_chains(sample_lists['data samples']))
Xs, block_im_dim = block_video(Xs, (48, 48), (3, 3))
to_video(Xs,
block_im_dim,
"A_TFD_KLD_CHAIN_VIDEO_{0:d}.avi".format(i),
frame_rate=10)
#sample_lists = GIP.sample_from_chain(Xd_batch[0,:].reshape((1,data_dim)), loop_iters=300, \
# sigma_scale=1.0)
#Xs = np.vstack(sample_lists["data samples"])
#file_name = "TFD_TEST_{0:d}.png".format(i)
#utils.visualize_samples(Xs, file_name, num_rows=15)
file_name = "A_TFD_KLD_PRIOR_SAMPLE.png"
Xs = OSM.sample_from_prior(20 * 20)
utils.visualize_samples(Xs, file_name, num_rows=20)
# test Parzen density estimator built from prior samples
# Xs = OSM.sample_from_prior(10000)
# [best_sigma, best_ll, best_lls] = \
# cross_validate_sigma(Xs, Xva, [0.09, 0.095, 0.1, 0.105, 0.11], 10)
# sort_idx = np.argsort(best_lls)
# sort_idx = sort_idx[0:400]
# utils.plot_line(np.arange(sort_idx.shape[0]), best_lls[sort_idx], "A_TFD_BEST_LLS_1.png")
# utils.visualize_samples(Xva[sort_idx], "A_TFD_BAD_FACES_1.png", num_rows=20)
return
def test_one_stage_model():
##########################
# Get some training data #
##########################
rng = np.random.RandomState(1234)
Xtr, Xva, Xte = load_binarized_mnist(data_path='./data/')
Xtr = np.vstack((Xtr, Xva))
Xva = Xte
#del Xte
tr_samples = Xtr.shape[0]
va_samples = Xva.shape[0]
batch_size = 128
batch_reps = 1
###############################################
# Setup some parameters for the OneStageModel #
###############################################
x_dim = Xtr.shape[1]
z_dim = 64
x_type = 'bernoulli'
xin_sym = T.matrix('xin_sym')
###############
# p_x_given_z #
###############
params = {}
shared_config = \
[ {'layer_type': 'fc',
'in_chans': z_dim,
'out_chans': 256,
'activation': relu_actfun,
'apply_bn': True}, \
{'layer_type': 'fc',
'in_chans': 256,
'out_chans': 7*7*128,
'activation': relu_actfun,
'apply_bn': True,
'shape_func_out': lambda x: T.reshape(x, (-1, 128, 7, 7))}, \
{'layer_type': 'conv',
'in_chans': 128, # in shape: (batch, 128, 7, 7)
'out_chans': 64, # out shape: (batch, 64, 14, 14)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'half',
'apply_bn': True} ]
output_config = \
[ {'layer_type': 'conv',
'in_chans': 64, # in shape: (batch, 64, 14, 14)
'out_chans': 1, # out shape: (batch, 1, 28, 28)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'half',
'apply_bn': False,
'shape_func_out': lambda x: T.flatten(x, 2)}, \
{'layer_type': 'conv',
'in_chans': 64,
'out_chans': 1,
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'half',
'apply_bn': False,
'shape_func_out': lambda x: T.flatten(x, 2)} ]
params['shared_config'] = shared_config
params['output_config'] = output_config
params['init_scale'] = 1.0
params['build_theano_funcs'] = False
p_x_given_z = HydraNet(rng=rng, Xd=xin_sym, \
params=params, shared_param_dicts=None)
p_x_given_z.init_biases(0.0)
###############
# q_z_given_x #
###############
params = {}
shared_config = \
[ {'layer_type': 'conv',
'in_chans': 1, # in shape: (batch, 784)
'out_chans': 64, # out shape: (batch, 64, 14, 14)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'double',
'apply_bn': True,
'shape_func_in': lambda x: T.reshape(x, (-1, 1, 28, 28))}, \
{'layer_type': 'conv',
'in_chans': 64, # in shape: (batch, 64, 14, 14)
'out_chans': 128, # out shape: (batch, 128, 7, 7)
'activation': relu_actfun,
'filt_dim': 5,
'conv_stride': 'double',
'apply_bn': True,
'shape_func_out': lambda x: T.flatten(x, 2)}, \
{'layer_type': 'fc',
'in_chans': 128*7*7,
'out_chans': 256,
'activation': relu_actfun,
'apply_bn': True} ]
output_config = \
[ {'layer_type': 'fc',
'in_chans': 256,
'out_chans': z_dim,
'activation': relu_actfun,
'apply_bn': False}, \
{'layer_type': 'fc',
'in_chans': 256,
'out_chans': z_dim,
'activation': relu_actfun,
'apply_bn': False} ]
params['shared_config'] = shared_config
params['output_config'] = output_config
params['init_scale'] = 1.0
params['build_theano_funcs'] = False
q_z_given_x = HydraNet(rng=rng, Xd=xin_sym, \
params=params, shared_param_dicts=None)
q_z_given_x.init_biases(0.0)
##############################################################
# Define parameters for the TwoStageModel, and initialize it #
##############################################################
print("Building the OneStageModel...")
osm_params = {}
osm_params['x_type'] = x_type
osm_params['obs_transform'] = 'sigmoid'
OSM = OneStageModel(rng=rng,
x_in=xin_sym,
x_dim=x_dim,
z_dim=z_dim,
p_x_given_z=p_x_given_z,
q_z_given_x=q_z_given_x,
params=osm_params)
################################################################
# Apply some updates, to check that they aren't totally broken #
################################################################
log_name = "{}_RESULTS.txt".format("OSM_TEST")
out_file = open(log_name, 'wb')
costs = [0. for i in range(10)]
learn_rate = 0.0005
momentum = 0.9
batch_idx = np.arange(batch_size) + tr_samples
for i in range(500000):
scale = min(0.5, ((i + 1) / 5000.0))
if (((i + 1) % 10000) == 0):
learn_rate = learn_rate * 0.95
# get the indices of training samples for this batch update
batch_idx += batch_size
if (np.max(batch_idx) >= tr_samples):
# we finished an "epoch", so we rejumble the training set
Xtr = row_shuffle(Xtr)
batch_idx = np.arange(batch_size)
Xb = to_fX(Xtr.take(batch_idx, axis=0))
#Xb = binarize_data(Xtr.take(batch_idx, axis=0))
# set sgd and objective function hyperparams for this update
OSM.set_sgd_params(lr=scale*learn_rate, \
mom_1=(scale*momentum), mom_2=0.98)
OSM.set_lam_nll(lam_nll=1.0)
OSM.set_lam_kld(lam_kld=1.0)
OSM.set_lam_l2w(1e-5)
# perform a minibatch update and record the cost for this batch
result = OSM.train_joint(Xb, batch_reps)
costs = [(costs[j] + result[j]) for j in range(len(result))]
if ((i % 500) == 0):
costs = [(v / 500.0) for v in costs]
str1 = "-- batch {0:d} --".format(i)
str2 = " joint_cost: {0:.4f}".format(costs[0])
str3 = " nll_cost : {0:.4f}".format(costs[1])
str4 = " kld_cost : {0:.4f}".format(costs[2])
str5 = " reg_cost : {0:.4f}".format(costs[3])
joint_str = "\n".join([str1, str2, str3, str4, str5])
print(joint_str)
out_file.write(joint_str + "\n")
out_file.flush()
costs = [0.0 for v in costs]
if (((i % 5000) == 0) or ((i < 10000) and ((i % 1000) == 0))):
# draw some independent random samples from the model
samp_count = 300
model_samps = OSM.sample_from_prior(samp_count)
file_name = "OSM_SAMPLES_b{0:d}.png".format(i)
utils.visualize_samples(model_samps, file_name, num_rows=15)
# compute free energy estimate for validation samples
Xva = row_shuffle(Xva)
fe_terms = OSM.compute_fe_terms(Xva[0:5000], 20)
fe_mean = np.mean(fe_terms[0]) + np.mean(fe_terms[1])
out_str = " nll_bound : {0:.4f}".format(fe_mean)
print(out_str)
out_file.write(out_str + "\n")
out_file.flush()
return
