def inner_fn(x_t, s_tm1, s_tm1_is):
x_1_t = x_1.fprop([x_t])
x_2_t = x_2.fprop([x_1_t])
x_3_t = x_3.fprop([x_2_t])
x_4_t = x_4.fprop([x_3_t])
x_5_t = x_5.fprop([x_4_t])
x_6_t = x_6.fprop([x_5_t])
phi_1_t = phi_1.fprop([x_6_t, s_tm1])
phi_2_t = phi_2.fprop([phi_1_t])
phi_3_t = phi_3.fprop([phi_2_t])
phi_4_t = phi_4.fprop([phi_3_t])
phi_mu_t = phi_mu.fprop([phi_4_t])
phi_sig_t = phi_sig.fprop([phi_4_t])
z_t = prior.fprop([phi_mu_t, phi_sig_t])
kl_t = kl.fprop([phi_mu_t, phi_sig_t])
z_1_t = z_1.fprop([z_t])
z_2_t = z_2.fprop([z_1_t])
z_3_t = z_3.fprop([z_2_t])
z_4_t = z_4.fprop([z_3_t])
theta_1_t = theta_1.fprop([z_4_t, s_tm1])
theta_2_t = theta_2.fprop([theta_1_t])
theta_3_t = theta_3.fprop([theta_2_t])
theta_4_t = theta_4.fprop([theta_3_t])
theta_mu_t = theta_mu.fprop([theta_4_t])
theta_sig_t = theta_sig.fprop([theta_4_t])
s_t = main_lstm.fprop([[x_6_t, z_4_t], [s_tm1]])
x_t_is = T.repeat(x_t, num_sample, axis=0)
x_1_t_is = x_1.fprop([x_t_is])
x_2_t_is = x_2.fprop([x_1_t_is])
x_3_t_is = x_3.fprop([x_2_t_is])
x_4_t_is = x_4.fprop([x_3_t_is])
x_5_t_is = x_5.fprop([x_4_t_is])
x_6_t_is = x_6.fprop([x_5_t_is])
phi_1_t_is = phi_1.fprop([x_6_t_is, s_tm1_is])
phi_2_t_is = phi_2.fprop([phi_1_t_is])
phi_3_t_is = phi_3.fprop([phi_2_t_is])
phi_4_t_is = phi_4.fprop([phi_3_t_is])
phi_mu_t_is = phi_mu.fprop([phi_4_t_is])
phi_sig_t_is = phi_sig.fprop([phi_4_t_is])
z_t_is = prior.sample([phi_mu_t_is, phi_sig_t_is])
z_1_t_is = z_1.fprop([z_t_is])
z_2_t_is = z_2.fprop([z_1_t_is])
z_3_t_is = z_3.fprop([z_2_t_is])
z_4_t_is = z_4.fprop([z_3_t_is])
prior_mu_t_is = T.zeros_like(z_t_is)
prior_sig_t_is = T.ones_like(z_t_is)
theta_1_t_is = theta_1.fprop([z_4_t_is, s_tm1_is])
theta_2_t_is = theta_2.fprop([theta_1_t_is])
theta_3_t_is = theta_3.fprop([theta_2_t_is])
theta_4_t_is = theta_4.fprop([theta_3_t_is])
theta_mu_t_is = theta_mu.fprop([theta_4_t_is])
theta_sig_t_is = theta_sig.fprop([theta_4_t_is])
mll = Gaussian(x_t_is, theta_mu_t_is, theta_sig_t_is) +\
Gaussian(z_t_is, prior_mu_t_is, prior_sig_t_is) -\
Gaussian(z_t_is, phi_mu_t_is, phi_sig_t_is)
mll = mll.reshape((batch_size, num_sample))
mll = logsumexp(-mll, axis=1) - T.log(num_sample)
s_t_is = main_lstm.fprop([[x_6_t_is, z_4_t_is], [s_tm1_is]])
return s_t, s_t_is, kl_t, theta_mu_t, theta_sig_t, mll
for k, v in updates.iteritems():
k.default_update = v
s_t = s_t[:-1]
s_shape = s_t.shape
s_in = T.concatenate([s_0, s_t.reshape((s_shape[0]*s_shape[1], -1))], axis=0)
theta_1_in = theta_1.fprop([s_in])
theta_2_in = theta_2.fprop([theta_1_in])
theta_3_in = theta_3.fprop([theta_2_in])
theta_4_in = theta_4.fprop([theta_3_in])
theta_mu_in = theta_mu.fprop([theta_4_in])
theta_sig_in = theta_sig.fprop([theta_4_in])
recon = Gaussian(x_in, theta_mu_in, theta_sig_in)
recon = recon.reshape((x_shape[0], x_shape[1]))
recon = recon * x_mask
recon_term = recon.sum(axis=0).mean()
recon_term.name = 'nll'
max_x = x.max()
mean_x = x.mean()
min_x = x.min()
max_x.name = 'max_x'
mean_x.name = 'mean_x'
min_x.name = 'min_x'
max_theta_mu = theta_mu_in.max()
mean_theta_mu = theta_mu_in.mean()
min_theta_mu = theta_mu_in.min()
max_theta_mu.name = 'max_theta_mu'
((s_t, s_t_is, kl_t, theta_mu_t, theta_sig_t, mll), updates) =\
theano.scan(fn=inner_fn,
sequences=[x],
outputs_info=[main_lstm.get_init_state(batch_size),
main_lstm.get_init_state(batch_size*num_sample),
None, None, None, None])
for k, v in updates.iteritems():
k.default_update = v
reshaped_x = x.reshape((x.shape[0]*x.shape[1], -1))
reshaped_theta_mu = theta_mu_t.reshape((theta_mu_t.shape[0]*theta_mu_t.shape[1], -1))
reshaped_theta_sig = theta_sig_t.reshape((theta_sig_t.shape[0]*theta_sig_t.shape[1], -1))
recon = Gaussian(reshaped_x, reshaped_theta_mu, reshaped_theta_sig)
recon = recon.reshape((theta_mu_t.shape[0], theta_mu_t.shape[1]))
recon = recon * x_mask
kl_t = kl_t * x_mask
recon_term = recon.sum(axis=0).mean()
kl_term = kl_t.sum(axis=0).mean()
nll_lower_bound = recon_term + kl_term
nll_lower_bound.name = 'nll_lower_bound'
mll = mll * x_mask
mll = -mll.sum(axis=0).mean()
mll.name = 'marginal_nll'
outputs = [mll, nll_lower_bound]
monitor_fn = theano.function(inputs=[x, x_mask],
outputs=outputs,
theta_2_in = theta_2.fprop([theta_1_in]) theta_3_in = theta_3.fprop([theta_2_in]) theta_4_in = theta_4.fprop([theta_3_in]) theta_mu_in = theta_mu.fprop([theta_4_in]) theta_sig_in = theta_sig.fprop([theta_4_in]) z_shape = phi_mu_t.shape phi_mu_in = phi_mu_t.reshape((z_shape[0]*z_shape[1], -1)) phi_sig_in = phi_sig_t.reshape((z_shape[0]*z_shape[1], -1)) prior_mu_in = prior_mu_t.reshape((z_shape[0]*z_shape[1], -1)) prior_sig_in = prior_sig_t.reshape((z_shape[0]*z_shape[1], -1)) kl_in = kl.fprop([phi_mu_in, phi_sig_in, prior_mu_in, prior_sig_in]) kl_t = kl_in.reshape((z_shape[0], z_shape[1])) recon = Gaussian(x_in, theta_mu_in, theta_sig_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon_term = recon.mean() kl_term = kl_t.mean() nll_lower_bound = recon_term + kl_term nll_lower_bound.name = 'nll_lower_bound' mn_x_shape = mn_x.shape mn_x_in = mn_x.reshape((mn_x_shape[0]*mn_x_shape[1], -1)) mn_x_1_in = x_1.fprop([mn_x_in]) mn_x_2_in = x_2.fprop([mn_x_1_in]) mn_x_3_in = x_3.fprop([mn_x_2_in]) mn_x_4_in = x_4.fprop([mn_x_3_in]) mn_x_4_in = mn_x_4_in.reshape((mn_x_shape[0], mn_x_shape[1], -1)) mn_s_0 = main_lstm.get_init_state(mn_batch_size) ((mn_s_t, mn_phi_mu_t, mn_phi_sig_t, mn_prior_mu_t, mn_prior_sig_t, mn_z_4_t), mn_updates) =\
def main(args):
trial = int(args['trial'])
pkl_name = 'rnn_gauss_%d' % trial
channel_name = 'valid_nll'
data_path = args['data_path']
save_path = args['save_path']
flgMSE = int(args['flgMSE'])
monitoring_freq = int(args['monitoring_freq'])
epoch = int(args['epoch'])
batch_size = int(args['batch_size'])
x_dim = int(args['x_dim'])
z_dim = int(args['z_dim'])
y_dim = int(args['y_dim'])
flgAgg = int(args['flgAgg'])
rnn_dim = int(args['rnn_dim'])
lr = float(args['lr'])
debug = int(args['debug'])
print "trial no. %d" % trial
print "batch size %d" % batch_size
print "learning rate %f" % lr
print "saving pkl file '%s'" % pkl_name
print "to the save path '%s'" % save_path
x2s_dim = 340
s2x_dim = 340
target_dim = k #x_dim - 1
model = Model()
train_data = UKdale(name='train',
prep='normalize',
cond=False,
path=data_path,
windows=windows,
appliances=appliances,
numApps=flgAgg,
period=period,
n_steps=n_steps,
stride_train=stride_train,
stride_test=stride_test)
X_mean = train_data.X_mean
X_std = train_data.X_std
valid_data = UKdale(name='valid',
prep='normalize',
cond=False,
path=data_path,
X_mean=X_mean,
X_std=X_std,
windows=windows,
appliances=appliances,
numApps=flgAgg,
period=period,
n_steps=n_steps,
stride_train=stride_train,
stride_test=stride_test)
init_W = InitCell('rand')
init_U = InitCell('ortho')
init_b = InitCell('zeros')
init_b_sig = InitCell('const', mean=0.6)
x, y = train_data.theano_vars()
if debug:
x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32)
temp = np.ones((15, batch_size), dtype=np.float32)
temp[:, -2:] = 0.
mask.tag.test_value = temp
x_1 = FullyConnectedLayer(name='x_1',
parent=['x_t'],
parent_dim=[x_dim],
nout=x2s_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
rnn = LSTM(name='rnn',
parent=['x_1'],
parent_dim=[x2s_dim],
nout=rnn_dim,
unit='tanh',
init_W=init_W,
init_U=init_U,
init_b=init_b)
theta_1 = FullyConnectedLayer(name='theta_1',
parent=['s_tm1'],
parent_dim=[rnn_dim],
nout=s2x_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
theta_mu = FullyConnectedLayer(name='theta_mu',
parent=['theta_1'],
parent_dim=[s2x_dim],
nout=target_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
theta_sig = FullyConnectedLayer(name='theta_sig',
parent=['theta_1'],
parent_dim=[s2x_dim],
nout=target_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
corr = FullyConnectedLayer(name='corr',
parent=['theta_1'],
parent_dim=[s2x_dim],
nout=1,
unit='tanh',
init_W=init_W,
init_b=init_b)
binary = FullyConnectedLayer(name='binary',
parent=['theta_1'],
parent_dim=[s2x_dim],
nout=1,
unit='sigmoid',
init_W=init_W,
init_b=init_b)
nodes = [rnn, x_1, theta_1, theta_mu, theta_sig] #, corr, binary
params = OrderedDict()
for node in nodes:
if node.initialize() is not None:
params.update(node.initialize())
params = init_tparams(params)
s_0 = rnn.get_init_state(batch_size)
x_1_temp = x_1.fprop([x], params)
def inner_fn(x_t, s_tm1):
s_t = rnn.fprop([[x_t], [s_tm1]], params)
theta_1_t = theta_1.fprop([s_t], params)
theta_mu_t = theta_mu.fprop([theta_1_t], params)
theta_sig_t = theta_sig.fprop([theta_1_t], params)
coeff_t = coeff.fprop([theta_1_t], params)
pred = Gaussian_sample(theta_mu_t, theta_sig_t)
return s_t, theta_mu_t, theta_sig_t, coeff_t, pred
((s_temp, theta_mu_temp, theta_sig_temp, coeff_temp, pred_temp),
updates) = theano.scan(fn=inner_fn,
sequences=[x_1_temp],
outputs_info=[s_0, None, None, None, None])
for k, v in updates.iteritems():
k.default_update = v
s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0)
'''
theta_1_temp = theta_1.fprop([s_temp], params)
theta_mu_temp = theta_mu.fprop([theta_1_temp], params)
theta_sig_temp = theta_sig.fprop([theta_1_temp], params)
corr_temp = corr.fprop([theta_1_temp], params)
binary_temp = binary.fprop([theta_1_temp], params)
'''
x_shape = x.shape
x_in = x.reshape((x_shape[0] * x_shape[1], -1))
theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1))
theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1))
corr_in = corr_temp.reshape((x_shape[0] * x_shape[1], -1))
binary_in = binary_temp.reshape((x_shape[0] * x_shape[1], -1))
if (flgAgg == -1):
prediction.name = 'x_reconstructed'
mse = T.mean((prediction - x)**2) # CHECK RESHAPE with an assertion
mae = T.mean(T.abs(prediction - x))
mse.name = 'mse'
pred_in = x.reshape((x_shape[0] * x_shape[1], -1))
else:
pred_temp = pred_temp.reshape((pred_temp.shape[0], pred_temp.shape[1]))
pred_temp.name = 'pred_' + str(flgAgg)
#y[:,:,flgAgg].reshape((y.shape[0],y.shape[1],1))
mse = T.mean((pred_temp - y.T)**2) # CHECK RESHAPE with an assertion
mae = T.mean(T.abs_(pred_temp - y.T))
mse.name = 'mse'
mae.name = 'mae'
pred_in = y.reshape((x.shape[0] * x.shape[1], -1), ndim=2)
recon = Gaussian(pred_in, theta_mu_in, theta_sig_in)
recon = recon.reshape((x_shape[0], x_shape[1]))
#recon = recon * mask
recon_term = recon.sum(axis=0).mean()
recon_term.name = 'nll'
max_x = x.max()
mean_x = x.mean()
min_x = x.min()
max_x.name = 'max_x'
mean_x.name = 'mean_x'
min_x.name = 'min_x'
max_theta_mu = theta_mu_in.max()
mean_theta_mu = theta_mu_in.mean()
min_theta_mu = theta_mu_in.min()
max_theta_mu.name = 'max_theta_mu'
mean_theta_mu.name = 'mean_theta_mu'
min_theta_mu.name = 'min_theta_mu'
max_theta_sig = theta_sig_in.max()
mean_theta_sig = theta_sig_in.mean()
min_theta_sig = theta_sig_in.min()
max_theta_sig.name = 'max_theta_sig'
mean_theta_sig.name = 'mean_theta_sig'
min_theta_sig.name = 'min_theta_sig'
model.inputs = [x, y]
model.params = params
model.nodes = nodes
optimizer = Adam(lr=lr)
extension = [
GradientClipping(batch_size=batch_size),
EpochCount(epoch),
Monitoring(freq=monitoring_freq,
ddout=[
recon_term, max_theta_sig, mean_theta_sig,
min_theta_sig, max_x, mean_x, min_x, max_theta_mu,
mean_theta_mu, min_theta_mu
],
data=[Iterator(valid_data, batch_size)]),
Picklize(freq=monitoring_freq, path=save_path),
EarlyStopping(freq=monitoring_freq,
path=save_path,
channel=channel_name),
WeightNorm()
]
mainloop = Training(name=pkl_name,
data=Iterator(train_data, batch_size),
model=model,
optimizer=optimizer,
cost=recon_term,
outputs=[recon_term],
extension=extension)
mainloop.run()
fLog = open(save_path + '/output.csv', 'w')
fLog.write("log,mse,mae\n")
for i, item in enumerate(mainloop.trainlog.monitor['nll_upper_bound']):
a = mainloop.trainlog.monitor['recon_term'][i]
d = mainloop.trainlog.monitor['mse'][i]
e = mainloop.trainlog.monitor['mae'][i]
fLog.write("{},{},{}\n".format(a, d, e))
def main(args):
theano.optimizer='fast_compile'
theano.config.exception_verbosity='high'
trial = int(args['trial'])
pkl_name = 'vrnn_gauss_%d' % trial
channel_name = 'valid_nll_upper_bound'
data_path = args['data_path']
save_path = args['save_path']
save_path = args['save_path']
period = int(args['period'])
n_steps = int(args['n_steps'])
stride_train = int(args['stride_train'])
stride_test = int(args['stride_test'])
monitoring_freq = int(args['monitoring_freq'])
epoch = int(args['epoch'])
batch_size = int(args['batch_size'])
x_dim = int(args['x_dim'])
z_dim = int(args['z_dim'])
rnn_dim = int(args['rnn_dim'])
lr = float(args['lr'])
debug = int(args['debug'])
print "trial no. %d" % trial
print "batch size %d" % batch_size
print "learning rate %f" % lr
print "saving pkl file '%s'" % pkl_name
print "to the save path '%s'" % save_path
q_z_dim = 150
p_z_dim = 150
p_x_dim = 250
x2s_dim = 10#250
z2s_dim = 10#150
target_dim = x_dim#(x_dim-1)
model = Model()
train_data = UKdale(name='train',
prep='none', #normalize
cond=False,
path=data_path,
period= period,
n_steps = n_steps,
x_dim=x_dim,
stride_train = stride_train,
stride_test = stride_test)
X_mean = train_data.X_mean
X_std = train_data.X_std
valid_data = UKdale(name='valid',
prep='none', #normalize
cond=False,
path=data_path,
X_mean=X_mean,
X_std=X_std)
init_W = InitCell('rand')
init_U = InitCell('ortho')
init_b = InitCell('zeros')
init_b_sig = InitCell('const', mean=0.6)
x, mask = train_data.theano_vars()
if debug:
x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32)
temp = np.ones((15, batch_size), dtype=np.float32)
temp[:, -2:] = 0.
mask.tag.test_value = temp
x_1 = FullyConnectedLayer(name='x_1',
parent=['x_t'],
parent_dim=[x_dim],
nout=x2s_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
z_1 = FullyConnectedLayer(name='z_1',
parent=['z_t'],
parent_dim=[z_dim],
nout=z2s_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
rnn = LSTM(name='rnn',
parent=['x_1', 'z_1'],
parent_dim=[x2s_dim, z2s_dim],
nout=rnn_dim,
unit='tanh',
init_W=init_W,
init_U=init_U,
init_b=init_b)
phi_1 = FullyConnectedLayer(name='phi_1', ## encoder
parent=['x_1', 's_tm1'],
parent_dim=[x2s_dim, rnn_dim],
nout=q_z_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
phi_mu = FullyConnectedLayer(name='phi_mu',
parent=['phi_1'],
parent_dim=[q_z_dim],
nout=z_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
phi_sig = FullyConnectedLayer(name='phi_sig',
parent=['phi_1'],
parent_dim=[q_z_dim],
nout=z_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
prior_1 = FullyConnectedLayer(name='prior_1',
parent=['s_tm1'],
parent_dim=[rnn_dim],
nout=p_z_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
prior_mu = FullyConnectedLayer(name='prior_mu',
parent=['prior_1'],
parent_dim=[p_z_dim],
nout=z_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
prior_sig = FullyConnectedLayer(name='prior_sig',
parent=['prior_1'],
parent_dim=[p_z_dim],
nout=z_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
theta_1 = FullyConnectedLayer(name='theta_1', ### decoder
parent=['z_1', 's_tm1'],
parent_dim=[z2s_dim, rnn_dim],
nout=p_x_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
theta_mu = FullyConnectedLayer(name='theta_mu',
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=target_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
theta_sig = FullyConnectedLayer(name='theta_sig',
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=target_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
corr = FullyConnectedLayer(name='corr', ## rho
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=1,
unit='tanh',
init_W=init_W,
init_b=init_b)
binary = FullyConnectedLayer(name='binary',
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=1,
unit='sigmoid',
init_W=init_W,
init_b=init_b)
nodes = [rnn,
x_1, z_1,
phi_1, phi_mu, phi_sig,
prior_1, prior_mu, prior_sig,
theta_1, theta_mu, theta_sig] #, corr, binary
params = OrderedDict()
for node in nodes:
if node.initialize() is not None:
params.update(node.initialize()) #Initialize values of the W matrices according to dim of parents
params = init_tparams(params)
s_0 = rnn.get_init_state(batch_size)
x_1_temp = x_1.fprop([x], params)
def inner_fn(x_t, s_tm1):
phi_1_t = phi_1.fprop([x_t, s_tm1], params)
phi_mu_t = phi_mu.fprop([phi_1_t], params)
phi_sig_t = phi_sig.fprop([phi_1_t], params)
prior_1_t = prior_1.fprop([s_tm1], params)
prior_mu_t = prior_mu.fprop([prior_1_t], params)
prior_sig_t = prior_sig.fprop([prior_1_t], params)
z_t = Gaussian_sample(phi_mu_t, phi_sig_t)
z_1_t = z_1.fprop([z_t], params)
s_t = rnn.fprop([[x_t, z_1_t], [s_tm1]], params)
return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_1_t
((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp, z_1_temp), updates) =\
theano.scan(fn=inner_fn,
sequences=[x_1_temp],
outputs_info=[s_0, None, None, None, None, None])
for k, v in updates.iteritems():
print("Update")
k.default_update = v
s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0)
s_temp.name = 'h_1'
z_1_temp.name = 'z_1'
theta_1_temp = theta_1.fprop([z_1_temp, s_temp], params)
theta_mu_temp = theta_mu.fprop([theta_1_temp], params)
theta_mu_temp.name = 'theta_mu'
theta_sig_temp = theta_sig.fprop([theta_1_temp], params)
theta_sig_temp.name = 'theta_sig'
#corr_temp = corr.fprop([theta_1_temp], params)
#corr_temp.name = 'corr'
#binary_temp = binary.fprop([theta_1_temp], params)
#binary_temp.name = 'binary'
kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp)
x_shape = x.shape
x_in = x.reshape((x_shape[0]*x_shape[1], -1))
theta_mu_in = theta_mu_temp.reshape((x_shape[0]*x_shape[1], -1))
theta_sig_in = theta_sig_temp.reshape((x_shape[0]*x_shape[1], -1))
#corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1))
#binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1))
recon = Gaussian(x_in, theta_mu_in, theta_sig_in) # BiGauss(x_in, theta_mu_in, theta_sig_in, corr_in, binary_in) # second term for the loss function
recon = recon.reshape((x_shape[0], x_shape[1]))
#recon = recon * mask
recon_term = recon.sum(axis=0).mean()
recon_term.name = 'recon_term'
#kl_temp = kl_temp * mask
kl_term = kl_temp.sum(axis=0).mean()
kl_term.name = 'kl_term'
nll_upper_bound = recon_term + kl_term
nll_upper_bound.name = 'nll_upper_bound'
max_x = x.max()
mean_x = x.mean()
min_x = x.min()
max_x.name = 'max_x'
mean_x.name = 'mean_x'
min_x.name = 'min_x'
max_theta_mu = theta_mu_in.max()
mean_theta_mu = theta_mu_in.mean()
min_theta_mu = theta_mu_in.min()
max_theta_mu.name = 'max_theta_mu'
mean_theta_mu.name = 'mean_theta_mu'
min_theta_mu.name = 'min_theta_mu'
max_theta_sig = theta_sig_in.max()
mean_theta_sig = theta_sig_in.mean()
min_theta_sig = theta_sig_in.min()
max_theta_sig.name = 'max_theta_sig'
mean_theta_sig.name = 'mean_theta_sig'
min_theta_sig.name = 'min_theta_sig'
max_phi_sig = phi_sig_temp.max()
mean_phi_sig = phi_sig_temp.mean()
min_phi_sig = phi_sig_temp.min()
max_phi_sig.name = 'max_phi_sig'
mean_phi_sig.name = 'mean_phi_sig'
min_phi_sig.name = 'min_phi_sig'
max_prior_sig = prior_sig_temp.max()
mean_prior_sig = prior_sig_temp.mean()
min_prior_sig = prior_sig_temp.min()
max_prior_sig.name = 'max_prior_sig'
mean_prior_sig.name = 'mean_prior_sig'
min_prior_sig.name = 'min_prior_sig'
prior_sig_output = prior_sig_temp
prior_sig_output.name = 'prior_sig_o'
phi_sig_output = phi_sig_temp
phi_sig_output.name = 'phi_sig_o'
model.inputs = [x, mask]
model.params = params
model.nodes = nodes
optimizer = Adam(
lr=lr
)
extension = [
GradientClipping(batch_size=batch_size),
EpochCount(epoch),
Monitoring(freq=monitoring_freq,
ddout=[nll_upper_bound, recon_term, kl_term,
max_phi_sig, mean_phi_sig, min_phi_sig,
max_prior_sig, mean_prior_sig, min_prior_sig,
max_theta_sig, mean_theta_sig, min_theta_sig,
max_x, mean_x, min_x,
max_theta_mu, mean_theta_mu, min_theta_mu, #0-16
#binary_temp, corr_temp,
theta_mu_temp, theta_sig_temp, #17-20
s_temp, z_1_temp
#phi_sig_output,phi_sig_output
],## added in order to explore the distributions
data=[Iterator(valid_data, batch_size)]),
Picklize(freq=monitoring_freq, path=save_path),
EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name),
WeightNorm()
]
mainloop = Training(
name=pkl_name,
data=Iterator(train_data, batch_size),
model=model,
optimizer=optimizer,
cost=nll_upper_bound,
outputs=[nll_upper_bound],
extension=extension
)
mainloop.run()
def main(args):
theano.optimizer = 'fast_compile'
theano.config.exception_verbosity = 'high'
trial = int(args['trial'])
pkl_name = 'vrnn_gauss_%d' % trial
channel_name = 'valid_nll_upper_bound'
data_path = args['data_path']
save_path = args['save_path']
save_path = args['save_path']
period = int(args['period'])
n_steps = int(args['n_steps'])
stride_train = int(args['stride_train'])
stride_test = int(args['stride_test'])
monitoring_freq = int(args['monitoring_freq'])
epoch = int(args['epoch'])
batch_size = int(args['batch_size'])
x_dim = int(args['x_dim'])
z_dim = int(args['z_dim'])
rnn_dim = int(args['rnn_dim'])
lr = float(args['lr'])
debug = int(args['debug'])
print "trial no. %d" % trial
print "batch size %d" % batch_size
print "learning rate %f" % lr
print "saving pkl file '%s'" % pkl_name
print "to the save path '%s'" % save_path
q_z_dim = 150
p_z_dim = 150
p_x_dim = 250
x2s_dim = 10 #250
z2s_dim = 10 #150
target_dim = x_dim #(x_dim-1)
model = Model()
Xtrain, ytrain, Xval, yval = fetch_ukdale(data_path,
windows,
appliances,
numApps=flgAgg,
period=period,
n_steps=n_steps,
stride_train=stride_train,
stride_test=stride_test)
train_data = UKdale(
name='train',
prep='normalize',
cond=True, # False
#path=data_path,
inputX=Xtrain,
labels=ytrain)
X_mean = train_data.X_mean
X_std = train_data.X_std
valid_data = UKdale(
name='valid',
prep='normalize',
cond=True, # False
#path=data_path,
X_mean=X_mean,
X_std=X_std,
inputX=Xval,
labels=yval)
init_W = InitCell('rand')
init_U = InitCell('ortho')
init_b = InitCell('zeros')
init_b_sig = InitCell('const', mean=0.6)
x, y = train_data.theano_vars()
if debug:
x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32)
temp = np.ones((15, batch_size), dtype=np.float32)
temp[:, -2:] = 0.
mask.tag.test_value = temp
x_1 = FullyConnectedLayer(
name='x_1',
parent=['x_t'], #OrderDict parent['x_t'] = x_dim
parent_dim=[x_dim],
nout=x2s_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
z_1 = FullyConnectedLayer(name='z_1',
parent=['z_t'],
parent_dim=[z_dim],
nout=z2s_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
rnn = LSTM(name='rnn',
parent=['x_1', 'z_1'],
parent_dim=[x2s_dim, z2s_dim],
nout=rnn_dim,
unit='tanh',
init_W=init_W,
init_U=init_U,
init_b=init_b)
phi_1 = FullyConnectedLayer(
name='phi_1', ## encoder
parent=['x_1', 's_tm1'],
parent_dim=[x2s_dim, rnn_dim],
nout=q_z_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
phi_mu = FullyConnectedLayer(name='phi_mu',
parent=['phi_1'],
parent_dim=[q_z_dim],
nout=z_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
phi_sig = FullyConnectedLayer(name='phi_sig',
parent=['phi_1'],
parent_dim=[q_z_dim],
nout=z_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
prior_1 = FullyConnectedLayer(name='prior_1',
parent=['s_tm1'],
parent_dim=[rnn_dim],
nout=p_z_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
prior_mu = FullyConnectedLayer(name='prior_mu',
parent=['prior_1'],
parent_dim=[p_z_dim],
nout=z_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
prior_sig = FullyConnectedLayer(name='prior_sig',
parent=['prior_1'],
parent_dim=[p_z_dim],
nout=z_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
theta_1 = FullyConnectedLayer(
name='theta_1', ### decoder
parent=['z_1', 's_tm1'],
parent_dim=[z2s_dim, rnn_dim],
nout=p_x_dim,
unit='relu',
init_W=init_W,
init_b=init_b)
theta_mu = FullyConnectedLayer(name='theta_mu',
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=target_dim,
unit='linear',
init_W=init_W,
init_b=init_b)
theta_sig = FullyConnectedLayer(name='theta_sig',
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=target_dim,
unit='softplus',
cons=1e-4,
init_W=init_W,
init_b=init_b_sig)
corr = FullyConnectedLayer(
name='corr', ## rho
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=1,
unit='tanh',
init_W=init_W,
init_b=init_b)
binary = FullyConnectedLayer(name='binary',
parent=['theta_1'],
parent_dim=[p_x_dim],
nout=1,
unit='sigmoid',
init_W=init_W,
init_b=init_b)
nodes = [
rnn, x_1, z_1, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig,
theta_1, theta_mu, theta_sig
] #, corr, binary
params = OrderedDict()
for node in nodes:
if node.initialize() is not None:
params.update(
node.initialize()
) #Initialize values of the W matrices according to dim of parents
params = init_tparams(params)
s_0 = rnn.get_init_state(batch_size)
x_1_temp = x_1.fprop([x], params)
def inner_fn(x_t, s_tm1):
phi_1_t = phi_1.fprop([x_t, s_tm1], params)
phi_mu_t = phi_mu.fprop([phi_1_t], params)
phi_sig_t = phi_sig.fprop([phi_1_t], params)
prior_1_t = prior_1.fprop([s_tm1], params)
prior_mu_t = prior_mu.fprop([prior_1_t], params)
prior_sig_t = prior_sig.fprop([prior_1_t], params)
z_t = Gaussian_sample(phi_mu_t, phi_sig_t)
z_1_t = z_1.fprop([z_t], params)
theta_1_t = theta_1.fprop([z_1_t, s_tm1], params)
theta_mu_t = theta_mu.fprop([theta_1_t], params)
theta_sig_t = theta_sig.fprop([theta_1_t], params)
pred = Gaussian_sample(theta_mu_t, theta_sig_t)
s_t = rnn.fprop([[x_t, z_1_t], [s_tm1]], params)
return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, pred
((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp, z_temp, z_1_temp, theta_1_temp, theta_mu_temp, theta_sig_temp, pred_temp), updates) =\
theano.scan(fn=inner_fn,
sequences=[x_1_temp], #non_sequences unchanging variables
#The tensor(s) to be looped over should be provided to scan using the sequence keyword argument
outputs_info=[s_0, None, None, None, None, None, None, None, None, None, None])#Initialization occurs in outputs_info
#=None This indicates to scan that it does not need to pass the prior result to _fn
'''
The general order of function parameters to:
sequences (if any), prior result(s) (if needed), non-sequences (if any)
'''
for k, v in updates.iteritems():
print("Update")
k.default_update = v
s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0)
s_temp.name = 'h_1' #gisse
z_temp.name = 'z'
z_1_temp.name = 'z_1' #gisse
#theta_1_temp = theta_1.fprop([z_1_temp, s_temp], params)
#theta_mu_temp = theta_mu.fprop([theta_1_temp], params)
theta_mu_temp.name = 'theta_mu'
#theta_sig_temp = theta_sig.fprop([theta_1_temp], params)
theta_sig_temp.name = 'theta_sig'
x_pred_temp.name = 'x_reconstructed'
#corr_temp = corr.fprop([theta_1_temp], params)
#corr_temp.name = 'corr'
#binary_temp = binary.fprop([theta_1_temp], params)
#binary_temp.name = 'binary'
if (flgAgg == -1):
prediction.name = 'x_reconstructed'
mse = T.mean((prediction - x)**2) # CHECK RESHAPE with an assertion
mae = T.mean(T.abs(prediction - x))
mse.name = 'mse'
pred_in = x.reshape((x_shape[0] * x_shape[1], -1))
else:
prediction.name = 'pred_' + str(flgAgg)
mse = T.mean((prediction - y[:, :, flgAgg].reshape(
(y.shape[0], y.shape[1],
1)))**2) # CHECK RESHAPE with an assertion
mae = T.mean(
T.abs_(prediction -
y[:, :, flgAgg].reshape((y.shape[0], y.shape[1], 1))))
mse.name = 'mse'
mae.name = 'mae'
pred_in = y[:, :, flgAgg].reshape((x.shape[0] * x.shape[1], -1),
ndim=2)
kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp,
prior_sig_temp)
#x_shape = x.shape
#x_in = x.reshape((x_shape[0]*x_shape[1], -1))
theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1))
theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1))
#corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1))
#binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1))
recon = Gaussian(
pred_in, theta_mu_in, theta_sig_in
) # BiGauss(x_in, theta_mu_in, theta_sig_in, corr_in, binary_in) # second term for the loss function
recon = recon.reshape((x_shape[0], x_shape[1]))
#recon = recon * mask
recon_term = recon.sum(axis=0).mean()
recon_term.name = 'recon_term'
#kl_temp = kl_temp * mask
kl_term = kl_temp.sum(axis=0).mean()
kl_term.name = 'kl_term'
nll_upper_bound = recon_term + kl_term
nll_upper_bound.name = 'nll_upper_bound'
max_x = x.max()
mean_x = x.mean()
min_x = x.min()
max_x.name = 'max_x'
mean_x.name = 'mean_x'
min_x.name = 'min_x'
max_theta_mu = theta_mu_in.max()
mean_theta_mu = theta_mu_in.mean()
min_theta_mu = theta_mu_in.min()
max_theta_mu.name = 'max_theta_mu'
mean_theta_mu.name = 'mean_theta_mu'
min_theta_mu.name = 'min_theta_mu'
max_theta_sig = theta_sig_in.max()
mean_theta_sig = theta_sig_in.mean()
min_theta_sig = theta_sig_in.min()
max_theta_sig.name = 'max_theta_sig'
mean_theta_sig.name = 'mean_theta_sig'
min_theta_sig.name = 'min_theta_sig'
max_phi_sig = phi_sig_temp.max()
mean_phi_sig = phi_sig_temp.mean()
min_phi_sig = phi_sig_temp.min()
max_phi_sig.name = 'max_phi_sig'
mean_phi_sig.name = 'mean_phi_sig'
min_phi_sig.name = 'min_phi_sig'
max_prior_sig = prior_sig_temp.max()
mean_prior_sig = prior_sig_temp.mean()
min_prior_sig = prior_sig_temp.min()
max_prior_sig.name = 'max_prior_sig'
mean_prior_sig.name = 'mean_prior_sig'
min_prior_sig.name = 'min_prior_sig'
prior_sig_output = prior_sig_temp
prior_sig_output.name = 'prior_sig_o'
phi_sig_output = phi_sig_temp
phi_sig_output.name = 'phi_sig_o'
model.inputs = [x, mask]
model.params = params
model.nodes = nodes
optimizer = Adam(lr=lr)
extension = [
GradientClipping(batch_size=batch_size),
EpochCount(epoch),
Monitoring(
freq=monitoring_freq,
ddout=[
nll_upper_bound,
recon_term,
kl_term,
mse,
mae,
max_phi_sig,
mean_phi_sig,
min_phi_sig,
max_prior_sig,
mean_prior_sig,
min_prior_sig,
max_theta_sig,
mean_theta_sig,
min_theta_sig,
max_x,
mean_x,
min_x,
max_theta_mu,
mean_theta_mu,
min_theta_mu, #0-17
#binary_temp, corr_temp,
theta_mu_temp,
theta_sig_temp, #17-20
s_temp,
z_temp,
z_1_temp,
x_pred_temp
#phi_sig_output,phi_sig_output
], ## added in order to explore the distributions
indexSep=22,
indexDDoutPlot=[(0, theta_mu_temp), (2, z_t_temp),
(3, prediction)],
instancesPlot=[0, 150], #, 80,150
savedFolder=save_path,
data=[Iterator(valid_data, batch_size)]),
Picklize(freq=monitoring_freq, path=save_path),
EarlyStopping(freq=monitoring_freq,
path=save_path,
channel=channel_name),
WeightNorm()
]
mainloop = Training(name=pkl_name,
data=Iterator(train_data, batch_size),
model=model,
optimizer=optimizer,
cost=nll_upper_bound,
outputs=[nll_upper_bound],
extension=extension)
mainloop.run()
fLog = open(save_path + '/output.csv', 'w')
fLog.write("log,kl,nll_upper_bound,mse,mae\n")
for i, item in enumerate(mainloop.trainlog.monitor['nll_upper_bound']):
a = mainloop.trainlog.monitor['recon_term'][i]
b = mainloop.trainlog.monitor['kl_term'][i]
c = mainloop.trainlog.monitor['nll_upper_bound'][i]
d = mainloop.trainlog.monitor['mse'][i]
e = mainloop.trainlog.monitor['mae'][i]
fLog.write("{},{},{},{},{}\n".format(a, b, c, d, e))
