def cost(self, X): if len(X) != 4: raise ValueError("The number of inputs does not match.") cost = GMM(X[0], X[1], X[2], X[3]) if self.use_sum: return cost.sum() else: return cost.mean()
((s_t, s_t_is, kl_t, theta_mu_t, theta_sig_t, coeff_t, mll), updates) =\ theano.scan(fn=inner_fn, sequences=[x], outputs_info=[main_lstm.get_init_state(batch_size), main_lstm.get_init_state(batch_size*num_sample), None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v reshaped_x = x.reshape((x.shape[0]*x.shape[1], -1)) reshaped_theta_mu = theta_mu_t.reshape((theta_mu_t.shape[0]*theta_mu_t.shape[1], -1)) reshaped_theta_sig = theta_sig_t.reshape((theta_sig_t.shape[0]*theta_sig_t.shape[1], -1)) reshaped_coeff = coeff_t.reshape((coeff_t.shape[0]*coeff_t.shape[1], -1)) recon = GMM(reshaped_x, reshaped_theta_mu, reshaped_theta_sig, reshaped_coeff) recon = recon.reshape((theta_mu_t.shape[0], theta_mu_t.shape[1])) recon_term = recon.sum(axis=0).mean() kl_term = kl_t.sum(axis=0).mean() nll_lower_bound = recon_term + kl_term nll_lower_bound.name = 'nll_lower_bound' mll = -mll.sum(axis=0).mean() mll.name = 'marginal_nll' outputs = [mll, nll_lower_bound] monitor_fn = theano.function(inputs=[x], outputs=outputs, on_unused_input='ignore', allow_input_downcast=True)
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'vrnn_gmm_%d' % trial channel_name = 'valid_nll_upper_bound' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) debug = int(args['debug']) num_sequences_per_batch = int(args['numSequences']) #based on appliance loadParam = args['loadAsKelly'] target_inclusion_prob = float(args['target_inclusion_prob']) loadAsKelly = True if (loadParam == 'N' or loadParam == 'n' or loadParam == 'no' or loadParam == 'NO' or loadParam == 'No'): loadAsKelly = False print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 100 #150 p_z_dim = 60 #150 p_x_dim = 20 #250 x2s_dim = 40 #250 z2s_dim = 40 #150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, reader = fetch_ukdale( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, flgAggSumScaled=1, flgFilterZeros=1, isKelly=loadAsKelly, seq_per_batch=num_sequences_per_batch, target_inclusion_prob=target_inclusion_prob) instancesPlot = { 0: [4, 20], 2: [5, 10] } #for now use hard coded instancesPlot for kelly sampling if (not loadAsKelly): instancesPlot = reader.build_dict_instances_plot( listDates, batch_size, Xval.shape[0]) train_data = UKdale( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = UKdale( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels=yval) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1', 'z_1'], parent_dim=[x2s_dim, z2s_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) ''' dissag_pred = FullyConnectedLayer(name='disag_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=num_apps, unit='relu', init_W=init_W, init_b=init_b) ''' phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1'], parent_dim=[x2s_dim, rnn_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[p_x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) nodes = [ rnn, x_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) def inner_fn(x_t, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) s_t = rnn.fprop([[x_t, z_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t, pred #, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp,z_t_temp, z_1_temp, theta_1_temp, theta_mu_temp, theta_sig_temp, coeff_temp, prediction), updates) =\ theano.scan(fn=inner_fn, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v s_temp = concatenate( [s_0[None, :, :], s_temp[:-1]], axis=0 ) # seems like this is for creating an additional dimension to s_0 ''' theta_1_temp = theta_1.fprop([z_1_temp, s_temp], params) theta_mu_temp = theta_mu.fprop([theta_1_temp], params) theta_sig_temp = theta_sig.fprop([theta_1_temp], params) coeff_temp = coeff.fprop([theta_1_temp], params) corr_temp = corr.fprop([theta_1_temp], params) binary_temp = binary.fprop([theta_1_temp], params) ''' s_temp.name = 'h_1' #gisse z_1_temp.name = 'z_1' #gisse z_t_temp.name = 'z' theta_mu_temp.name = 'theta_mu_temp' theta_sig_temp.name = 'theta_sig_temp' coeff_temp.name = 'coeff' #corr_temp.name = 'corr' #binary_temp.name = 'binary' if (flgAgg == -1): prediction.name = 'x_reconstructed' mse = T.mean((prediction - x)**2) # CHECK RESHAPE with an assertion mae = T.mean(T.abs(prediction - x)) mse.name = 'mse' pred_in = x.reshape((x_shape[0] * x_shape[1], -1)) else: prediction.name = 'pred_' + str(flgAgg) #[:,:,flgAgg].reshape((y.shape[0],y.shape[1],1) mse = T.mean( (prediction - y)**2) # As axis = None is calculated for all mae = T.mean(T.abs_(prediction - y)) mse.name = 'mse' mae.name = 'mae' pred_in = y.reshape((y.shape[0] * y.shape[1], -1)) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0] * x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) recon = GMM( pred_in, theta_mu_in, theta_sig_in, coeff_in ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon.name = 'gmm_out' #recon = recon * mask recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' #kl_temp = kl_temp * mask kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' nll_upper_bound = recon_term + kl_term #+ mse if (flgMSE): nll_upper_bound = nll_upper_bound + mse nll_upper_bound.name = 'nll_upper_bound' max_x = x.max() mean_x = x.mean() min_x = x.min() max_x.name = 'max_x' mean_x.name = 'mean_x' min_x.name = 'min_x' max_theta_mu = theta_mu_in.max() mean_theta_mu = theta_mu_in.mean() min_theta_mu = theta_mu_in.min() max_theta_mu.name = 'max_theta_mu' mean_theta_mu.name = 'mean_theta_mu' min_theta_mu.name = 'min_theta_mu' max_theta_sig = theta_sig_in.max() mean_theta_sig = theta_sig_in.mean() min_theta_sig = theta_sig_in.min() max_theta_sig.name = 'max_theta_sig' mean_theta_sig.name = 'mean_theta_sig' min_theta_sig.name = 'min_theta_sig' coeff_max = coeff_in.max() coeff_min = coeff_in.min() coeff_mean_max = coeff_in.mean(axis=0).max() coeff_mean_min = coeff_in.mean(axis=0).min() coeff_max.name = 'coeff_max' coeff_min.name = 'coeff_min' coeff_mean_max.name = 'coeff_mean_max' coeff_mean_min.name = 'coeff_mean_min' max_phi_sig = phi_sig_temp.max() mean_phi_sig = phi_sig_temp.mean() min_phi_sig = phi_sig_temp.min() max_phi_sig.name = 'max_phi_sig' mean_phi_sig.name = 'mean_phi_sig' min_phi_sig.name = 'min_phi_sig' max_prior_sig = prior_sig_temp.max() mean_prior_sig = prior_sig_temp.mean() min_prior_sig = prior_sig_temp.min() max_prior_sig.name = 'max_prior_sig' mean_prior_sig.name = 'mean_prior_sig' min_prior_sig.name = 'min_prior_sig' model.inputs = [x, mask, y, y_mask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch), Monitoring( freq=monitoring_freq, ddout=[ nll_upper_bound, recon_term, kl_term, mse, mae, theta_mu_temp, theta_sig_temp, z_t_temp, prediction, #corr_temp, binary_temp, s_temp, z_1_temp ], indexSep=5, indexDDoutPlot=[(0, theta_mu_temp), (2, z_t_temp), (3, prediction)], instancesPlot=instancesPlot, #{0:[4,20],2:[5,10]},#, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = {0: lr, 15: (lr / 10), 70: (lr / 100)} mainloop = Training(name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[nll_upper_bound], extension=extension, lr_iterations=lr_iterations) #mainloop.run() fLog = open(save_path + '/output.csv', 'w') fLog.write("log,kl,nll_upper_bound,mse,mae\n") for i, item in enumerate(mainloop.trainlog.monitor['nll_upper_bound']): a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] c = mainloop.trainlog.monitor['nll_upper_bound'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{},{},{},{},{}\n".format(a, b, c, d, e))
def main(args): trial = int(args['trial']) pkl_name = 'vrnn_gmm_%d' % trial channel_name = 'valid_nll_upper_bound' data_path = args['data_path'] save_path = args['save_path'] monitoring_freq = int(args['monitoring_freq']) force_saving_freq = int(args['force_saving_freq']) reset_freq = int(args['reset_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) m_batch_size = int(args['m_batch_size']) x_dim = int(args['x_dim']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) lr = float(args['lr']) debug = int(args['debug']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 500 p_z_dim = 500 p_x_dim = 500 x2s_dim = 500 z2s_dim = 500 target_dim = x_dim * k file_name = 'blizzard_unseg_tbptt' normal_params = np.load(data_path + file_name + '_normal.npz') X_mean = normal_params['X_mean'] X_std = normal_params['X_std'] model = Model() train_data = Blizzard_tbptt(name='train', path=data_path, frame_size=x_dim, file_name=file_name, X_mean=X_mean, X_std=X_std) valid_data = Blizzard_tbptt(name='valid', path=data_path, frame_size=x_dim, file_name=file_name, X_mean=X_mean, X_std=X_std) x = train_data.theano_vars() m_x = valid_data.theano_vars() if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=theano.config.floatX) m_x.tag.test_value = np.zeros((15, m_batch_size, x_dim), dtype=theano.config.floatX) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) x_2 = FullyConnectedLayer(name='x_2', parent=['x_1'], parent_dim=[x2s_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) x_3 = FullyConnectedLayer(name='x_3', parent=['x_2'], parent_dim=[x2s_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) x_4 = FullyConnectedLayer(name='x_4', parent=['x_3'], parent_dim=[x2s_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_2 = FullyConnectedLayer(name='z_2', parent=['z_1'], parent_dim=[z2s_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_3 = FullyConnectedLayer(name='z_3', parent=['z_2'], parent_dim=[z2s_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_4 = FullyConnectedLayer(name='z_4', parent=['z_3'], parent_dim=[z2s_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_4', 'z_4'], parent_dim=[x2s_dim, z2s_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_4', 's_tm1'], parent_dim=[x2s_dim, rnn_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_2 = FullyConnectedLayer(name='phi_2', parent=['phi_1'], parent_dim=[q_z_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_3 = FullyConnectedLayer(name='phi_3', parent=['phi_2'], parent_dim=[q_z_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_4 = FullyConnectedLayer(name='phi_4', parent=['phi_3'], parent_dim=[q_z_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_4'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_4'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_2 = FullyConnectedLayer(name='prior_2', parent=['prior_1'], parent_dim=[p_z_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_3 = FullyConnectedLayer(name='prior_3', parent=['prior_2'], parent_dim=[p_z_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_4 = FullyConnectedLayer(name='prior_4', parent=['prior_3'], parent_dim=[p_z_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_4'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_4'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_4', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_2 = FullyConnectedLayer(name='theta_2', parent=['theta_1'], parent_dim=[p_x_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_3 = FullyConnectedLayer(name='theta_3', parent=['theta_2'], parent_dim=[p_x_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_4 = FullyConnectedLayer(name='theta_4', parent=['theta_3'], parent_dim=[p_x_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_4'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_4'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_4'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) nodes = [rnn, x_1, x_2, x_3, x_4, z_1, z_2, z_3, z_4, phi_1, phi_2, phi_3, phi_4, phi_mu, phi_sig, prior_1, prior_2, prior_3, prior_4, prior_mu, prior_sig, theta_1, theta_2, theta_3, theta_4, theta_mu, theta_sig, coeff] params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) step_count = sharedX(0, name='step_count') last_rnn = np.zeros((batch_size, rnn_dim*2), dtype=theano.config.floatX) rnn_tm1 = sharedX(last_rnn, name='rnn_tm1') shared_updates = OrderedDict() shared_updates[step_count] = step_count + 1 s_0 = T.switch(T.eq(T.mod(step_count, reset_freq), 0), rnn.get_init_state(batch_size), rnn_tm1) x_shape = x.shape x_in = x.reshape((x_shape[0]*x_shape[1], -1)) x_1_in = x_1.fprop([x_in], params) x_2_in = x_2.fprop([x_1_in], params) x_3_in = x_3.fprop([x_2_in], params) x_4_in = x_4.fprop([x_3_in], params) x_4_in = x_4_in.reshape((x_shape[0], x_shape[1], -1)) def inner_fn(x_t, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1], params) phi_2_t = phi_2.fprop([phi_1_t], params) phi_3_t = phi_3.fprop([phi_2_t], params) phi_4_t = phi_4.fprop([phi_3_t], params) phi_mu_t = phi_mu.fprop([phi_4_t], params) phi_sig_t = phi_sig.fprop([phi_4_t], params) prior_1_t = prior_1.fprop([s_tm1], params) prior_2_t = prior_2.fprop([prior_1_t], params) prior_3_t = prior_3.fprop([prior_2_t], params) prior_4_t = prior_4.fprop([prior_3_t], params) prior_mu_t = prior_mu.fprop([prior_4_t], params) prior_sig_t = prior_sig.fprop([prior_4_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) z_2_t = z_2.fprop([z_1_t], params) z_3_t = z_3.fprop([z_2_t], params) z_4_t = z_4.fprop([z_3_t], params) s_t = rnn.fprop([[x_t, z_4_t], [s_tm1]], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_4_t ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp, z_4_temp), updates) =\ theano.scan(fn=inner_fn, sequences=[x_4_in], outputs_info=[s_0, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v shared_updates[rnn_tm1] = s_temp[-1] s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0) theta_1_temp = theta_1.fprop([z_4_temp, s_temp], params) theta_2_temp = theta_2.fprop([theta_1_temp], params) theta_3_temp = theta_3.fprop([theta_2_temp], params) theta_4_temp = theta_4.fprop([theta_3_temp], params) theta_mu_temp = theta_mu.fprop([theta_4_temp], params) theta_sig_temp = theta_sig.fprop([theta_4_temp], params) coeff_temp = coeff.fprop([theta_4_temp], params) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape x_in = x.reshape((x_shape[0]*x_shape[1], -1)) theta_mu_in = theta_mu_temp.reshape((x_shape[0]*x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0]*x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0]*x_shape[1], -1)) recon = GMM(x_in, theta_mu_in, theta_sig_in, coeff_in) recon_term = recon.mean() kl_term = kl_temp.mean() nll_upper_bound = recon_term + kl_term nll_upper_bound.name = 'nll_upper_bound' m_x_1_temp = x_1.fprop([m_x], params) m_x_2_temp = x_2.fprop([m_x_1_temp], params) m_x_3_temp = x_3.fprop([m_x_2_temp], params) m_x_4_temp = x_4.fprop([m_x_3_temp], params) m_s_0 = rnn.get_init_state(m_batch_size) ((m_s_temp, m_phi_mu_temp, m_phi_sig_temp, m_prior_mu_temp, m_prior_sig_temp, m_z_4_temp), m_updates) =\ theano.scan(fn=inner_fn, sequences=[m_x_4_temp], outputs_info=[m_s_0, None, None, None, None, None]) for k, v in m_updates.iteritems(): k.default_update = v m_s_temp = concatenate([m_s_0[None, :, :], m_s_temp[:-1]], axis=0) m_theta_1_temp = theta_1.fprop([m_z_4_temp, m_s_temp], params) m_theta_2_temp = theta_2.fprop([m_theta_1_temp], params) m_theta_3_temp = theta_3.fprop([m_theta_2_temp], params) m_theta_4_temp = theta_4.fprop([m_theta_3_temp], params) m_theta_mu_temp = theta_mu.fprop([m_theta_4_temp], params) m_theta_sig_temp = theta_sig.fprop([m_theta_4_temp], params) m_coeff_temp = coeff.fprop([m_theta_4_temp], params) m_kl_temp = KLGaussianGaussian(m_phi_mu_temp, m_phi_sig_temp, m_prior_mu_temp, m_prior_sig_temp) m_x_shape = m_x.shape m_x_in = m_x.reshape((m_x_shape[0]*m_x_shape[1], -1)) m_theta_mu_in = m_theta_mu_temp.reshape((m_x_shape[0]*m_x_shape[1], -1)) m_theta_sig_in = m_theta_sig_temp.reshape((m_x_shape[0]*m_x_shape[1], -1)) m_coeff_in = m_coeff_temp.reshape((m_x_shape[0]*m_x_shape[1], -1)) m_recon = GMM(m_x_in, m_theta_mu_in, m_theta_sig_in, m_coeff_in) m_recon_term = m_recon.mean() m_kl_term = m_kl_temp.mean() m_nll_upper_bound = m_recon_term + m_kl_term m_nll_upper_bound.name = 'nll_upper_bound' m_recon_term.name = 'recon_term' m_kl_term.name = 'kl_term' max_x = m_x.max() mean_x = m_x.mean() min_x = m_x.min() max_x.name = 'max_x' mean_x.name = 'mean_x' min_x.name = 'min_x' max_theta_mu = m_theta_mu_in.max() mean_theta_mu = m_theta_mu_in.mean() min_theta_mu = m_theta_mu_in.min() max_theta_mu.name = 'max_theta_mu' mean_theta_mu.name = 'mean_theta_mu' min_theta_mu.name = 'min_theta_mu' max_theta_sig = m_theta_sig_in.max() mean_theta_sig = m_theta_sig_in.mean() min_theta_sig = m_theta_sig_in.min() max_theta_sig.name = 'max_theta_sig' mean_theta_sig.name = 'mean_theta_sig' min_theta_sig.name = 'min_theta_sig' max_phi_sig = m_phi_sig_temp.max() mean_phi_sig = m_phi_sig_temp.mean() min_phi_sig = m_phi_sig_temp.min() max_phi_sig.name = 'max_phi_sig' mean_phi_sig.name = 'mean_phi_sig' min_phi_sig.name = 'min_phi_sig' max_prior_sig = m_prior_sig_temp.max() mean_prior_sig = m_prior_sig_temp.mean() min_prior_sig = m_prior_sig_temp.min() max_prior_sig.name = 'max_prior_sig' mean_prior_sig.name = 'mean_prior_sig' min_prior_sig.name = 'min_prior_sig' model.inputs = [x] model.params = params model.nodes = nodes model.set_updates(shared_updates) optimizer = Adam( lr=lr ) monitor_fn = theano.function(inputs=[m_x], outputs=[m_nll_upper_bound, m_recon_term, m_kl_term, max_phi_sig, mean_phi_sig, min_phi_sig, max_prior_sig, mean_prior_sig, min_prior_sig, max_theta_sig, mean_theta_sig, min_theta_sig, max_x, mean_x, min_x, max_theta_mu, mean_theta_mu, min_theta_mu], on_unused_input='ignore') extension = [ GradientClipping(batch_size=batch_size, check_nan=1), EpochCount(epoch), Monitoring(freq=monitoring_freq, monitor_fn=monitor_fn, ddout=[m_nll_upper_bound, m_recon_term, m_kl_term, max_phi_sig, mean_phi_sig, min_phi_sig, max_prior_sig, mean_prior_sig, min_prior_sig, max_theta_sig, mean_theta_sig, min_theta_sig, max_x, mean_x, min_x, max_theta_mu, mean_theta_mu, min_theta_mu], data=[Iterator(train_data, m_batch_size, start=0, end=112640), Iterator(valid_data, m_batch_size, start=2040064, end=2152704)]), Picklize(freq=monitoring_freq, force_save_freq=force_saving_freq, path=save_path), EarlyStopping(freq=monitoring_freq, force_save_freq=force_saving_freq, path=save_path, channel=channel_name), WeightNorm() ] mainloop = Training( name=pkl_name, data=Iterator(train_data, batch_size, start=0, end=2040064), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[nll_upper_bound], extension=extension ) mainloop.run()
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'dp_dis1-sch_%d' % trial channel_name = 'mae' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) typeLoad = int(args['typeLoad']) debug = int(args['debug']) kSchedSamp = int(args['kSchedSamp']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 150 p_z_dim = 150 p_x_dim = 150 #250 x2s_dim = 100 #250 y2s_dim = 100 z2s_dim = 100 #150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_dataport( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, trainPer=0.6, valPer=0.2, testPer=0.2, typeLoad=typeLoad, flgAggSumScaled=1, flgFilterZeros=1) print(reader.stdTrain, reader.meanTrain) instancesPlot = { 0: [4], 2: [10] } #for now use hard coded instancesPlot for kelly sampling train_data = Dataport( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels=yval) test_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xtest, labels=ytest) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() scheduleSamplingMask = T.fvector('schedMask') x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp """ print (mainloop) attrs = vars(mainloop) print ', '.join("%s: %s" % item for item in attrs.items()) print (type(mainloop.model.nodes[1])) print (type(mainloop.model.params))""" #for node in mainloop.model.nodes: # print(node.name) ''' for node in mainloop.model.nodes: print("Name:", node.name, "Parent:", node.parent, "Unit:",node.unit,"init_W:", node.init_W, "init_b:", node.init_b) for param in mainloop.model.params: print(type(param),param) ''' """rnn = LSTM(name='rnn', parent=['x_1', 'z_1','y_1'], parent_dim=[x2s_dim, z2s_dim, y_dim], nout=rnn_dim, unit='tanh', init_W=mainloop.model.nodes[0].init_W, init_U=mainloop.model.nodes[0].init_U, init_b=mainloop.model.nodes[0].init_b) x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=mainloop.model.nodes[1].init_W, init_b=mainloop.model.nodes[1].init_b) y_1 = FullyConnectedLayer(name='y_1', parent=['y_t'], parent_dim=[y_dim], nout=y2s_dim, unit='relu', init_W=mainloop.model.nodes[2].init_W, init_b=mainloop.model.nodes[2].init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=mainloop.model.nodes[3].init_W, init_b=mainloop.model.nodes[3].init_b) phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1','y_1'], parent_dim=[x2s_dim, rnn_dim,y2s_dim], nout=q_z_dim, unit='relu', init_W=mainloop.model.nodes[4].init_W, init_b=mainloop.model.nodes[4].init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=mainloop.model.nodes[5].init_W, init_b=mainloop.model.nodes[5].init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=mainloop.model.nodes[6].init_W, init_b=mainloop.model.nodes[6].init_b) prior_1 = FullyConnectedLayer(name='prior_1', parent=['x_1','s_tm1'], parent_dim=[x2s_dim,rnn_dim], nout=p_z_dim, unit='relu', init_W=mainloop.model.nodes[7].init_W, init_b=mainloop.model.nodes[7].init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=mainloop.model.nodes[8].init_W, init_b=mainloop.model.nodes[8].init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=mainloop.model.nodes[9].init_W, init_b=mainloop.model.nodes[9].init_b) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=mainloop.model.nodes[10].init_W, init_b=mainloop.model.nodes[10].init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=mainloop.model.nodes[11].init_W, init_b=mainloop.model.nodes[11].init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=mainloop.model.nodes[12].init_W, init_b=mainloop.model.nodes[12].init_b) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=mainloop.model.nodes[13].init_W, init_b=mainloop.model.nodes[13].init_b)""" #pickle is from experiment gmmAE/18-05-30_16-07_app3 fmodel = open('dp_dis1-sch_1.pkl', 'rb') mainloop = cPickle.load(fmodel) fmodel.close() rnn = mainloop.model.nodes[0] x_1 = mainloop.model.nodes[1] y_1 = mainloop.model.nodes[2] z_1 = mainloop.model.nodes[3] phi_1 = mainloop.model.nodes[4] phi_mu = mainloop.model.nodes[5] phi_sig = mainloop.model.nodes[6] prior_1 = mainloop.model.nodes[7] prior_mu = mainloop.model.nodes[8] prior_sig = mainloop.model.nodes[9] theta_1 = mainloop.model.nodes[10] theta_mu = mainloop.model.nodes[11] theta_sig = mainloop.model.nodes[12] coeff = mainloop.model.nodes[13] nodes = [ rnn, x_1, y_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = mainloop.model.params s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) y_1_temp = y_1.fprop([y], params) def inner_fn_val(x_t, s_tm1): prior_1_t = prior_1.fprop([x_t, s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) pred_1_t = y_1.fprop([pred_t], params) s_t = rnn.fprop([[x_t, z_1_t, pred_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, prior_mu_t, prior_sig_t, theta_mu_t, theta_sig_t, coeff_t, pred_t #, y_pred #corr_temp, binary_temp ((s_temp_val, prior_mu_temp_val, prior_sig_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val, prediction_val), updates_val) =\ theano.scan(fn=inner_fn_val, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v s_temp_val = concatenate([s_0[None, :, :], s_temp_val[:-1]], axis=0) def inner_fn_train(x_t, y_t, schedSampMask, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1, y_t], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([x_t, s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) if (schedSampMask == 1): s_t = rnn.fprop([[x_t, z_1_t, y_t], [s_tm1]], params) else: y_t_aux = y_1.fprop([pred], params) s_t = rnn.fprop([[x_t, z_1_t, y_t_aux], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, theta_mu_t, theta_sig_t, coeff_t, pred #, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp, theta_mu_temp, theta_sig_temp, coeff_temp, prediction), updates) =\ theano.scan(fn=inner_fn_train, sequences=[x_1_temp, y_1_temp, scheduleSamplingMask], outputs_info=[s_0, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v #s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0)# seems like this is for creating an additional dimension to s_0 theta_mu_temp.name = 'theta_mu_temp' theta_sig_temp.name = 'theta_sig_temp' coeff_temp.name = 'coeff' if (flgAgg == -1): prediction.name = 'x_reconstructed' mse = T.mean((prediction - x)**2) # CHECK RESHAPE with an assertion mae = T.mean(T.abs(prediction - x)) mse.name = 'mse' pred_in = x.reshape((x_shape[0] * x_shape[1], -1)) else: prediction.name = 'pred_' + str(flgAgg) mse = T.mean( (prediction - y)**2) # As axis = None is calculated for all mae = T.mean(T.abs_(prediction - y)) mse.name = 'mse' mae.name = 'mae' pred_in = y.reshape((y.shape[0] * y.shape[1], -1)) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0] * x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) recon = GMM( pred_in, theta_mu_in, theta_sig_in, coeff_in ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon.name = 'gmm_out' recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' nll_upper_bound = recon_term + kl_term #+ mse if (flgMSE): nll_upper_bound = nll_upper_bound + mse nll_upper_bound.name = 'nll_upper_bound' ######################## TEST (GENERATION) TIME prediction_val.name = 'generated__' + str(flgAgg) mse_val = T.mean( (prediction_val - y)**2) # As axis = None is calculated for all mae_val = T.mean(T.abs_(prediction_val - y)) #y_unNormalize = (y * reader.stdTrain) + reader.meanTrain # accessing to just an scalar when loading y_dim=1 #prediction_valAux = (prediction_val * reader.stdTrain) + reader.meanTrain #mse_valUnNorm = T.mean((prediction_valAux - y_unNormalize)**2) # As axis = None is calculated for all #mae_valUnNorm = T.mean( T.abs_(prediction_valAux - y_unNormalize) ) mse_val.name = 'mse_val' mae_val.name = 'mae_val' pred_in_val = y.reshape((y.shape[0] * y.shape[1], -1)) theta_mu_in_val = theta_mu_temp_val.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in_val = theta_sig_temp_val.reshape( (x_shape[0] * x_shape[1], -1)) coeff_in_val = coeff_temp_val.reshape((x_shape[0] * x_shape[1], -1)) recon_val = GMM( pred_in_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((x_shape[0], x_shape[1])) recon_val.name = 'gmm_out_val' recon_term_val = recon_val.sum(axis=0).mean() recon_term_val.name = 'recon_term_val' model.inputs = [x, mask, y, y_mask, scheduleSamplingMask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) header = "epoch,log,kl,nll_upper_bound,mse,mae\n" extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch, save_path, header), Monitoring( freq=monitoring_freq, ddout=[nll_upper_bound, recon_term, kl_term, mse, mae, prediction], indexSep=5, instancesPlot=instancesPlot, #{0:[4,20],2:[5,10]},#, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = {0: lr, 75: (lr / 10), 150: (lr / 100)} """mainloop = Training( name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[recon_term, kl_term, nll_upper_bound, mse, mae], n_steps = n_steps, extension=extension, lr_iterations=lr_iterations, k_speedOfconvergence=kSchedSamp )""" """mainloop.restore( data=Iterator(train_data, batch_size), cost=nll_upper_bound, model=model, optimizer=mainloop.optimizer )""" mainloop.restore(name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[recon_term, kl_term, nll_upper_bound, mse, mae], n_steps=n_steps, extension=extension, lr_iterations=lr_iterations, k_speedOfconvergence=kSchedSamp) mainloop.run() data = Iterator(test_data, batch_size) test_fn = theano.function( inputs=[x, y], #[x, y], #givens={x:Xtest}, #on_unused_input='ignore', #z=( ,200,1) allow_input_downcast=True, outputs=[prediction_val, recon_term_val, mse_val, mae_val] #prediction_val, mse_val, mae_val , updates= updates_val #, allow_input_downcast=True, on_unused_input='ignore' ) testOutput = [] numBatchTest = 0 for batch in data: outputGeneration = test_fn(batch[0], batch[2]) #(20, 220, 1) testOutput.append(outputGeneration[1:]) # outputGeneration[0].shape #(20, 220, 40) #if (numBatchTest<5): ''' plt.figure(1) plt.plot(np.transpose(outputGeneration[0],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_z_0-4".format(numBatchTest)) plt.clf() plt.figure(2) plt.plot(np.transpose(outputGeneration[1],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_s_0-4".format(numBatchTest)) plt.clf() plt.figure(3) plt.plot(np.transpose(outputGeneration[2],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_theta_0-4".format(numBatchTest)) plt.clf() ''' plt.figure(4) plt.plot(np.transpose(outputGeneration[0], [1, 0, 2])[4]) plt.plot(np.transpose(batch[2], [1, 0, 2])[4]) plt.savefig( save_path + "/vrnn_dis_generated{}_RealAndPred_0-4".format(numBatchTest)) plt.clf() plt.figure(4) plt.plot(np.transpose(batch[0], [1, 0, 2])[4]) plt.savefig(save_path + "/vrnn_dis_generated{}_Realagg_0-4".format(numBatchTest)) plt.clf() numBatchTest += 1 testOutput = np.asarray(testOutput) print(testOutput.shape) recon_test = testOutput[:, 0].mean() mse_test = testOutput[:, 1].mean() mae_test = testOutput[:, 2].mean() #mseUnNorm_test = testOutput[:, 3].mean() #maeUnNorm_test = testOutput[:, 4].mean() fLog = open(save_path + '/output.csv', 'w') fLog.write(str(lr_iterations) + "\n") fLog.write(str(windows) + "\n") fLog.write("logTest,mseTest,maeTest, mseTestUnNorm, maeTestUnNorm\n") fLog.write("{},{},{}\n".format(recon_test, mse_test, mae_test)) fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{},{}\n".format(q_z_dim, p_z_dim, p_x_dim, x2s_dim, y2s_dim, z2s_dim)) header = "epoch,log,kl,mse,mae\n" fLog.write(header) for i, item in enumerate(mainloop.trainlog.monitor['recon_term']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{:d},{:.2f},{:.2f},{:.3f},{:.3f}\n".format(f, a, b, d, e))
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'vrnn_gmm_%d' % trial channel_name = 'nll_upper_bound' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) typeLoad = int(args['typeLoad']) debug = int(args['debug']) n_steps_val = n_steps print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path print str(windows) q_z_dim = 180 p_z_dim = 180 p_x_dim = 200 x2s_dim = 100 z2s_dim = 150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_dataport( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, trainPer=0.6, valPer=0.2, testPer=0.2, typeLoad=typeLoad, flgAggSumScaled=1, flgFilterZeros=1) instancesPlot = { 0: [5] } #for now use hard coded instancesPlot for kelly sampling train_data = Dataport( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=ytrain, labels=Xtrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=yval, labels=Xval) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() scheduleSamplingMask = T.fvector('schedMask') x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1', 'z_1'], parent_dim=[x2s_dim, z2s_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) ''' dissag_pred = FullyConnectedLayer(name='disag_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=num_apps, unit='relu', init_W=init_W, init_b=init_b) ''' phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1'], parent_dim=[x2s_dim, rnn_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[p_x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) nodes = [ rnn, x_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) def inner_val_fn(s_tm1): ''' phi_1_t = phi_1.fprop([x_t, s_tm1], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) ''' prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) pred_1_t = x_1.fprop([pred_t], params) s_t = rnn.fprop([[pred_1_t, z_1_t], [s_tm1]], params) return s_t, pred_t, z_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t # prior_mu_temp_val, prior_sig_temp_val ((s_temp_val, prediction_val, z_t_temp_val, theta_1_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val), updates_val) =\ theano.scan(fn=inner_val_fn , n_steps=n_steps_val, #already 1 subtracted if doing next step outputs_info=[s_0, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v def inner_fn(x_t, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) s_t = rnn.fprop([[x_t, z_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t, pred #, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp,z_t_temp, z_1_temp, theta_1_temp, theta_mu_temp, theta_sig_temp, coeff_temp, prediction), updates) =\ theano.scan(fn=inner_fn, sequences=[x_1_temp ], outputs_info=[s_0, None, None, None, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v s_temp = concatenate( [s_0[None, :, :], s_temp[:-1]], axis=0 ) # seems like this is for creating an additional dimension to s_0 ''' theta_1_temp = theta_1.fprop([z_1_temp, s_temp], params) theta_mu_temp = theta_mu.fprop([theta_1_temp], params) theta_sig_temp = theta_sig.fprop([theta_1_temp], params) coeff_temp = coeff.fprop([theta_1_temp], params) corr_temp = corr.fprop([theta_1_temp], params) binary_temp = binary.fprop([theta_1_temp], params) ''' s_temp.name = 'h' #gisse z_1_temp.name = 'z2' #gisse z_t_temp.name = 'z' theta_mu_temp.name = 'mu' theta_sig_temp.name = 'sig' coeff_temp.name = 'coeff' prediction.name = 'Prediction-' + str(appliances[flgAgg][:-1]) mse = T.mean((prediction - x)**2) # As axis = None is calculated for all mae = T.mean(T.abs_(prediction - x)) mse.name = 'mse' mae.name = 'mae' x_in = x.reshape((batch_size * n_steps, -1)) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0] * x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) recon = GMM( x_in, theta_mu_in, theta_sig_in, coeff_in ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon.name = 'gmm_out' #recon = recon * mask recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' #kl_temp = kl_temp * mask kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' nll_upper_bound = recon_term + kl_term #+ mse if (flgMSE): nll_upper_bound = nll_upper_bound + mse nll_upper_bound.name = 'nll_upper_bound' ############## TEST ############### theta_mu_in_val = theta_mu_temp_val.reshape((batch_size * n_steps, -1)) theta_sig_in_val = theta_sig_temp_val.reshape((batch_size * n_steps, -1)) coeff_in_val = coeff_temp_val.reshape((batch_size * n_steps, -1)) pred_in = prediction_val.reshape((batch_size * n_steps, -1)) recon_val = GMM( pred_in, theta_mu_in_val, theta_sig_in_val, coeff_in_val ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((batch_size, n_steps)) recon_val.name = 'gmm_out_val' model.inputs = [x, mask, y, y_mask, scheduleSamplingMask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) header = "epoch,log,kl,nll_upper_bound,mse,mae\n" extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch, save_path, header), Monitoring( freq=monitoring_freq, ddout=[nll_upper_bound, recon_term, kl_term, mse, mae, prediction], indexSep=5, indexDDoutPlot=[(0, theta_mu_temp), (2, z_t_temp), (3, prediction)], instancesPlot=instancesPlot, #{0:[4,20],2:[5,10]},#, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = { 0: lr, 30: (lr / 10) } #, 150:(lr/10), 270:(lr/100), 370:(lr/1000) mainloop = Training(name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[nll_upper_bound], n_steps=n_steps, extension=extension, lr_iterations=lr_iterations) mainloop.run() test_fn = theano.function( inputs=[], outputs=[prediction_val, recon_val], updates= updates_val #, allow_input_downcast=True, on_unused_input='ignore' ) outputGeneration = test_fn() #{0:[4,20], 2:[5,10]} ''' plt.figure(1) plt.plot(np.transpose(outputGeneration[0],[1,0,2])[5]) plt.savefig(save_path+"/vrnn_dis_generated_z_0-4.ps") plt.figure(2) plt.plot(np.transpose(outputGeneration[1],[1,0,2])[5]) plt.savefig(save_path+"/vrnn_dis_generated_s_0-4.ps") plt.figure(3) plt.plot(np.transpose(outputGeneration[2],[1,0,2])[5]) plt.savefig(save_path+"/vrnn_dis_generated_theta_0-4.ps") ''' plt.figure(1) plt.plot(np.transpose(outputGeneration[0], [1, 0, 2])[2]) plt.savefig(save_path + "/vrnn_dis_generated_pred_0-2.ps") plt.figure(2) plt.plot(np.transpose(outputGeneration[0], [1, 0, 2])[10]) plt.savefig(save_path + "/vrnn_dis_generated_pred_0-10.ps") plt.figure(3) plt.plot(np.transpose(outputGeneration[0], [1, 0, 2])[15]) plt.savefig(save_path + "/vrnn_dis_generated_pred_0-15.ps") testLogLike = np.asarray(outputGeneration[1]).mean() fLog = open(save_path + '/output.csv', 'w') fLog.write(str(lr_iterations) + "\n") fLog.write(str(windows) + "\n") fLog.write("Test-log-likelihood\n") fLog.write("{}\n".format(testLogLike)) fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{}\n".format(q_z_dim, p_z_dim, p_x_dim, x2s_dim, z2s_dim)) fLog.write("epoch,log,kl,nll_upper_bound,mse,mae\n") for i, item in enumerate(mainloop.trainlog.monitor['nll_upper_bound']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] c = mainloop.trainlog.monitor['nll_upper_bound'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{:d},{:.2f},{:.2f},{:.2f},{:.3f},{:.3f}\n".format( f, a, b, c, d, e)) fLog.close() f = open(save_path + '/outputRealGeneration.pkl', 'wb') cPickle.dump(outputGeneration, f, -1) f.close()
def main(args): theano.optimizer = 'fast_compile' theano.config.exception_verbosity = 'high' trial = int(args['trial']) pkl_name = 'rnn_gmm_%d' % trial channel_name = 'valid_nll' data_path = args['data_path'] save_path = args['save_path'] flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) z_dim = int(args['z_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) lr = float(args['lr']) debug = int(args['debug']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path x2s_dim = 50 #300 s2x_dim = 50 #300 target_dim = k #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval = fetch_ukdale(data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test) print("Inside: ", Xtrain.shape, ytrain.shape, Xval.shape, yval.shape) train_data = UKdale( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = UKdale( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels=yval) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() #mask, y_mask ''' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp ''' x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1'], parent_dim=[x2s_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) theta_1 = FullyConnectedLayer(name='theta_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=s2x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[s2x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[s2x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[s2x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) ''' corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[s2x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[s2x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) ''' nodes = [rnn, x_1, theta_1, theta_mu, theta_sig, coeff] params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) s_0 = rnn.get_init_state(batch_size) x_shape = x.shape x_1_temp = x_1.fprop([x], params) def inner_fn(x_t, s_tm1): s_t = rnn.fprop([[x_t], [s_tm1]], params) theta_1_t = theta_1.fprop([s_t], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) return s_t, theta_mu_t, theta_sig_t, coeff_t, pred ((s_temp, theta_mu_temp, theta_sig_temp, coeff_temp, pred_temp), updates) = theano.scan(fn=inner_fn, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0) ''' theta_1_temp = theta_1.fprop([s_temp], params) theta_mu_temp = theta_mu.fprop([theta_1_temp], params) theta_sig_temp = theta_sig.fprop([theta_1_temp], params) coeff_temp = coeff.fprop([theta_1_temp], params) corr_temp = corr.fprop([theta_1_temp], params) binary_temp = binary.fprop([theta_1_temp], params) ''' x_shape = x.shape x_in = x.reshape((x_shape[0] * x_shape[1], -1)) theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0] * x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) if (flgAgg == -1): pred_temp.name = 'x_reconstructed' mse = T.mean((pred_temp - x)**2) # CHECK RESHAPE with an assertion mae = T.mean(T.abs(pred_temp - x)) mse.name = 'mse' pred_in = x.reshape((x_shape[0] * x_shape[1], -1)) else: #pred_temp = pred_temp.reshape((pred_temp.shape[0], pred_temp.shape[1])) pred_temp.name = 'pred_' + str(flgAgg) #y[:,:,flgAgg].reshape((y.shape[0],y.shape[1],1)) mse = T.mean((pred_temp - y)**2) # CHECK RESHAPE with an assertion mae = T.mean(T.abs_(pred_temp - y)) mse.name = 'mse' mae.name = 'mae' pred_in = y.reshape((y.shape[0] * y.shape[1], -1)) recon = GMM(pred_in, theta_mu_in, theta_sig_in, coeff_in) #, binary_in recon.name = 'recon' recon = recon.reshape((y.shape[0], y.shape[1])) #recon = recon * y_mask #(200, 1000), (1000, 200) recon_term = recon.sum(axis=0).mean() recon_term.name = 'nll' max_x = x.max() mean_x = x.mean() min_x = x.min() max_x.name = 'max_x' mean_x.name = 'mean_x' min_x.name = 'min_x' max_theta_mu = theta_mu_in.max() mean_theta_mu = theta_mu_in.mean() min_theta_mu = theta_mu_in.min() max_theta_mu.name = 'max_theta_mu' mean_theta_mu.name = 'mean_theta_mu' min_theta_mu.name = 'min_theta_mu' ''' max_theta_sig = theta_sig_in.max() mean_theta_sig = theta_sig_in.mean() min_theta_sig = theta_sig_in.min() max_theta_sig.name = 'max_theta_sig' mean_theta_sig.name = 'mean_theta_sig' min_theta_sig.name = 'min_theta_sig' coeff_max = coeff_in.max() coeff_min = coeff_in.min() coeff_mean_max = coeff_in.mean(axis=0).max() coeff_mean_min = coeff_in.mean(axis=0).min() coeff_max.name = 'coeff_max' coeff_min.name = 'coeff_min' coeff_mean_max.name = 'coeff_mean_max' coeff_mean_min.name = 'coeff_mean_min' ''' model.inputs = [x, mask, y, y_mask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch), Monitoring( freq=monitoring_freq, ddout=[ recon_term, mse, mae, #max_theta_sig, mean_theta_sig, min_theta_sig, max_x, mean_x, min_x, max_theta_mu, mean_theta_mu, min_theta_mu, #coeff_max, coeff_min, coeff_mean_max, coeff_mean_min,#16 theta_mu_temp, theta_sig_temp, pred_temp, coeff_temp, s_temp ], indexSep=9, indexDDoutPlot=[(0, theta_mu_temp), (2, pred_temp)], instancesPlot=[10, 100], #, 80,150 savedFolder=save_path, data=[Iterator(valid_data, batch_size)]), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = {0: lr, 20: (lr / 10), 150: (lr / 100), 200: (lr / 1000)} mainloop = Training(name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=recon_term, outputs=[recon_term], extension=extension, lr_iterations=lr_iterations) mainloop.run() fLog = open(save_path + '/output.csv', 'w') fLog.write("log,mse,mae\n") for i, item in enumerate(mainloop.trainlog.monitor['nll']): a = mainloop.trainlog.monitor['nll'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{},{},{}\n".format(a, d, e))
for k, v in updates.iteritems(): k.default_update = v s_t = s_t[:-1] s_shape = s_t.shape s_in = T.concatenate([s_0, s_t.reshape((s_shape[0]*s_shape[1], -1))], axis=0) theta_1_in = theta_1.fprop([s_in]) theta_2_in = theta_2.fprop([theta_1_in]) theta_3_in = theta_3.fprop([theta_2_in]) theta_4_in = theta_4.fprop([theta_3_in]) theta_mu_in = theta_mu.fprop([theta_4_in]) theta_sig_in = theta_sig.fprop([theta_4_in]) coeff_in = coeff.fprop([theta_4_in]) recon = GMM(x_in, theta_mu_in, theta_sig_in, coeff_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon = recon * x_mask recon_term = recon.sum(axis=0).mean() recon_term.name = 'nll' max_x = x.max() mean_x = x.mean() min_x = x.min() max_x.name = 'max_x' mean_x.name = 'mean_x' min_x.name = 'min_x' max_theta_mu = theta_mu_in.max() mean_theta_mu = theta_mu_in.mean() min_theta_mu = theta_mu_in.min()
theta_2_in = theta_2.fprop([theta_1_in]) theta_3_in = theta_3.fprop([theta_2_in]) theta_4_in = theta_4.fprop([theta_3_in]) theta_mu_in = theta_mu.fprop([theta_4_in]) theta_sig_in = theta_sig.fprop([theta_4_in]) coeff_in = coeff.fprop([theta_4_in]) z_shape = phi_mu_t.shape phi_mu_in = phi_mu_t.reshape((z_shape[0]*z_shape[1], -1)) phi_sig_in = phi_sig_t.reshape((z_shape[0]*z_shape[1], -1)) prior_mu_in = prior_mu_t.reshape((z_shape[0]*z_shape[1], -1)) prior_sig_in = prior_sig_t.reshape((z_shape[0]*z_shape[1], -1)) kl_in = kl.fprop([phi_mu_in, phi_sig_in, prior_mu_in, prior_sig_in]) kl_t = kl_in.reshape((z_shape[0], z_shape[1])) recon = GMM(x_in, theta_mu_in, theta_sig_in, coeff_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon_term = recon.mean() kl_term = kl_t.mean() nll_lower_bound = recon_term + kl_term nll_lower_bound.name = 'nll_lower_bound' mn_x_shape = mn_x.shape mn_x_in = mn_x.reshape((mn_x_shape[0]*mn_x_shape[1], -1)) mn_x_1_in = x_1.fprop([mn_x_in]) mn_x_2_in = x_2.fprop([mn_x_1_in]) mn_x_3_in = x_3.fprop([mn_x_2_in]) mn_x_4_in = x_4.fprop([mn_x_3_in]) mn_x_4_in = mn_x_4_in.reshape((mn_x_shape[0], mn_x_shape[1], -1)) mn_s_0 = main_lstm.get_init_state(mn_batch_size)
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) #pkl_name = 'dp_dis1-nosch_%d' % trial channel_name = 'mae' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") testRef_file = args['testRef_file'] flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) typeLoad = int(args['typeLoad']) debug = int(args['debug']) kSchedSamp = int(args['kSchedSamp']) clipped = int(args['clipped']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "Reading pkl file '%s'" % testRef_file print "to the save path '%s'" % save_path q_z_dim = 150 p_z_dim = 150 p_x_dim = 150 #250 x2s_dim = 100 #250 y2s_dim = 100 z2s_dim = 100 #150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_dataport( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, trainPer=0.6, valPer=0.2, testPer=0.2, typeLoad=typeLoad, flgAggSumScaled=1, flgFilterZeros=1) print(reader.stdTrain, reader.meanTrain) instancesPlot = { 0: [4], 2: [5] } #for now use hard coded instancesPlot for kelly sampling train_data = Dataport( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels=yval) test_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xtest, labels=ytest) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = test_data.theano_vars() scheduleSamplingMask = T.fvector('schedMask') x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp fmodel = open(testRef_file, 'rb') mainloop = cPickle.load(fmodel) fmodel.close() #define layers rnn = mainloop.model.nodes[0] x_1 = mainloop.model.nodes[1] y_1 = mainloop.model.nodes[2] z_1 = mainloop.model.nodes[3] phi_1 = mainloop.model.nodes[4] phi_mu = mainloop.model.nodes[5] phi_sig = mainloop.model.nodes[6] prior_1 = mainloop.model.nodes[7] prior_mu = mainloop.model.nodes[8] prior_sig = mainloop.model.nodes[9] theta_1 = mainloop.model.nodes[10] theta_mu = mainloop.model.nodes[11] theta_sig = mainloop.model.nodes[12] coeff = mainloop.model.nodes[13] nodes = [ rnn, x_1, y_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = mainloop.model.params """params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params)""" s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) def inner_fn_val(x_t, s_tm1): prior_1_t = prior_1.fprop([x_t, s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) pred_1_t = y_1.fprop([pred_t], params) s_t = rnn.fprop([[x_t, z_1_t, pred_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, prior_mu_t, prior_sig_t, theta_mu_t, theta_sig_t, coeff_t, pred_t #, y_pred #corr_temp, binary_temp ((s_temp_val, prior_mu_temp_val, prior_sig_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val, prediction_val), updates_val) =\ theano.scan(fn=inner_fn_val, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v s_temp_val = concatenate([s_0[None, :, :], s_temp_val[:-1]], axis=0) x_shape = x.shape ######################## TEST (GENERATION) TIME prediction_val = T.clip(prediction_val, 0.0, np.inf) prediction_val.name = 'generated__' + str(flgAgg) mse_val = T.mean( (prediction_val - y)**2) # As axis = None is calculated for all mae_val = T.mean(T.abs_(prediction_val - y)) mse_val.name = 'mse_val' mae_val.name = 'mae_val' pred_in_val = y.reshape((y.shape[0] * y.shape[1], -1)) theta_mu_in_val = theta_mu_temp_val.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in_val = theta_sig_temp_val.reshape( (x_shape[0] * x_shape[1], -1)) coeff_in_val = coeff_temp_val.reshape((x_shape[0] * x_shape[1], -1)) recon_val = GMM( pred_in_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((x_shape[0], x_shape[1])) recon_val.name = 'gmm_out_val' recon_term_val = recon_val.sum(axis=0).mean() recon_term_val.name = 'recon_term_val' data = Iterator(test_data, batch_size) test_fn = theano.function( inputs=[x, y], #[x, y], allow_input_downcast=True, outputs=[ prediction_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val, recon_term_val ] #prediction_val, mse_val, mae_val , updates= updates_val #, allow_input_downcast=True, on_unused_input='ignore' ) testOutput = [] bestInstsancesPred = [] bestInstsancesDisa = [] bestInstsancesAggr = [] numBatchTest = 0 for batch in data: #, recon_term_val, mse_val, mae_val x_test, y_test = batch[0], batch[2] outputGeneration = test_fn(x_test, y_test) #(20, 220, 1) #testOutput.append(outputGeneration[1:]) # outputGeneration[0].shape #(20, 220, 40) if (clipped == 1): prediction_test = outputGeneration[0].clip(min=0) theta_mu_test = outputGeneration[1] theta_sig_test = outputGeneration[2] coeff_in_test = outputGeneration[3] realAggTest = np.transpose(x_test, [1, 0, 2]) realDisagTest = np.transpose( y_test, [1, 0, 2]) # because y_test already transformed prediction_test = np.transpose(prediction_test, [1, 0, 2]) ### test loglike y_test = y_test.reshape((y_test.shape[0] * y_test.shape[1], -1)) recon_test = GMM_outside( y_test, theta_mu_test, theta_sig_test, coeff_in_test ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_test = recon_test.reshape((x_test.shape[0], x_test.shape[1])) recon_term_test = recon_test.sum(axis=0).mean() ## test mse - mae mse_test = np.mean( (prediction_test - realDisagTest)**2) # As axis = None is calculated for all mae_test = np.mean(np.absolute(prediction_test - realDisagTest)) testOutput.append( (recon_term_test, mse_test, mae_test, outputGeneration[4])) batchMae_test = np.mean(np.absolute(prediction_test - realDisagTest), axis=(1, 2)) idxMin = np.argmin(batchMae_test) for idx in np.asarray(idxMin).reshape(1, -1)[0, :]: plt.figure(1) plt.plot(prediction_test[idx]) plt.plot(realDisagTest[idx]) plt.legend(["Predicted", "Real"]) plt.savefig( save_path + "/vrnn_dis1_test-b{}_Pred-Real_0-{}".format(numBatchTest, idx)) plt.clf() plt.figure(3) plt.plot(np.transpose(batch[0], [1, 0, 2])[idx]) plt.savefig( save_path + "/vrnn_dis1_test-b{}_RealAgg_0-{}".format(numBatchTest, idx)) plt.clf() bestInstsancesPred.append(prediction_test[idx]) bestInstsancesDisa.append(realDisagTest[idx]) bestInstsancesAggr.append(realAggTest[idx]) numBatchTest += 1 testOutput = np.asarray(testOutput) recon_testOutside = np.mean(testOutput[:, 0]) mse_test = np.mean(testOutput[:, 1]) mae_test = np.mean(testOutput[:, 2]) recon_testInside = np.mean(testOutput[:, 3]) #mseUnNorm_test = testOutput[:, 3].mean() #maeUnNorm_test = testOutput[:, 4].mean() fLog = open(save_path + '/output_test.csv', 'w') fLog.write(str(windows) + "\n") fLog.write( "logTestOutside,logTestInside,mseTest,maeTest, mseTestUnNorm, maeTestUnNorm\n" ) fLog.write("{},{},{},{}\n".format(recon_testOutside, recon_testInside, mse_test, mae_test)) fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{},{}\n".format(q_z_dim, p_z_dim, p_x_dim, x2s_dim, y2s_dim, z2s_dim))
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'dp_dis1-sch_%d' % trial channel_name = 'mae' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) typeLoad = int(args['typeLoad']) debug = int(args['debug']) kSchedSamp = int(args['kSchedSamp']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 150 p_z_dim = 150 p_x_dim = 150 #250 x2s_dim = 100 #250 y2s_dim = 100 z2s_dim = 100 #150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_dataport( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, trainPer=0.6, valPer=0.2, testPer=0.2, typeLoad=typeLoad, flgAggSumScaled=1, flgFilterZeros=1) print(reader.stdTrain, reader.meanTrain) instancesPlot = { 0: [4], 2: [10] } #for now use hard coded instancesPlot for kelly sampling train_data = Dataport( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels=yval) test_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xtest, labels=ytest) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() scheduleSamplingMask = T.fvector('schedMask') x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp fmodel = open('dp_dis1-sch_1_best.pkl', 'rb') mainloop = cPickle.load(fmodel) fmodel.close() #attrs = vars(mainloop) #print ', '.join("%s: %s" % item for item in attrs.items()) """names = [x.name for x in mainloop.model.nodes] print(names) print(mainloop.model.nodes)""" #define layers rnn = mainloop.model.nodes[0] x_1 = mainloop.model.nodes[1] y_1 = mainloop.model.nodes[2] z_1 = mainloop.model.nodes[3] phi_1 = mainloop.model.nodes[4] phi_mu = mainloop.model.nodes[5] phi_sig = mainloop.model.nodes[6] prior_1 = mainloop.model.nodes[7] prior_mu = mainloop.model.nodes[8] prior_sig = mainloop.model.nodes[9] theta_1 = mainloop.model.nodes[10] theta_mu = mainloop.model.nodes[11] theta_sig = mainloop.model.nodes[12] coeff = mainloop.model.nodes[13] nodes = [ rnn, x_1, y_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = mainloop.model.params """params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize())""" #params = init_tparams(params) """OrderedDict([('W_x_1__rnn', W_x_1__rnn), ('W_z_1__rnn', W_z_1__rnn), ('W_y_1__rnn', W_y_1__rnn), ('U_rnn__rnn', U_rnn__rnn), ('b_rnn', b_rnn), ('W_x_t__x_1', W_x_t__x_1), ('b_x_1', b_x_1), ('W_y_t__y_1', W_y_t__y_1), ('b_y_1', b_y_1), ('W_z_t__z_1', W_z_t__z_1), ('b_z_1', b_z_1), ('W_x_1__phi_1', W_x_1__phi_1), ('W_s_tm1__phi_1', W_s_tm1__phi_1), ('W_y_1__phi_1', W_y_1__phi_1), ('b_phi_1', b_phi_1), ('W_phi_1__phi_mu', W_phi_1__phi_mu), ('b_phi_mu', b_phi_mu), ('W_phi_1__phi_sig', W_phi_1__phi_sig), ('b_phi_sig', b_phi_sig), ('W_x_1__prior_1', W_x_1__prior_1), ('W_s_tm1__prior_1', W_s_tm1__prior_1), ('b_prior_1', b_prior_1), ('W_prior_1__prior_mu', W_prior_1__prior_mu), ('b_prior_mu', b_prior_mu), ('W_prior_1__prior_sig', W_prior_1__prior_sig), ('b_prior_sig', b_prior_sig), ('W_z_1__theta_1', W_z_1__theta_1), ('W_s_tm1__theta_1', W_s_tm1__theta_1), ('b_theta_1', b_theta_1), ('W_theta_1__theta_mu', W_theta_1__theta_mu), ('b_theta_mu', b_theta_mu), ('W_theta_1__theta_sig', W_theta_1__theta_sig), ('b_theta_sig', b_theta_sig), ('W_theta_1__coeff', W_theta_1__coeff), ('b_coeff', b_coeff)])""" """w1 = params['W_x_1__rnn'] w2 = params['W_phi_1__phi_sig'] w3 = params['W_theta_1__coeff'] print("Initialized W matricies:") print("W1:",w1.get_value()) print("W2:",w2.get_value()) print("W3:",w3.get_value()) print("\n\n--------------------------------\n\n")""" s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) y_1_temp = y_1.fprop([y], params) def inner_fn_val(x_t, s_tm1): prior_1_t = prior_1.fprop([x_t, s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) pred_1_t = y_1.fprop([pred_t], params) s_t = rnn.fprop([[x_t, z_1_t, pred_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, prior_mu_t, prior_sig_t, theta_mu_t, theta_sig_t, coeff_t, pred_t #, y_pred #corr_temp, binary_temp ((s_temp_val, prior_mu_temp_val, prior_sig_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val, prediction_val), updates_val) =\ theano.scan(fn=inner_fn_val, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v s_temp_val = concatenate([s_0[None, :, :], s_temp_val[:-1]], axis=0) x_shape = x.shape ######################## TEST (GENERATION) TIME prediction_val.name = 'generated__' + str(flgAgg) mse_val = T.mean( (prediction_val - y)**2) # As axis = None is calculated for all mae_val = T.mean(T.abs_(prediction_val - y)) mse_val.name = 'mse_val' mae_val.name = 'mae_val' pred_in_val = y.reshape((y.shape[0] * y.shape[1], -1)) theta_mu_in_val = theta_mu_temp_val.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in_val = theta_sig_temp_val.reshape( (x_shape[0] * x_shape[1], -1)) coeff_in_val = coeff_temp_val.reshape((x_shape[0] * x_shape[1], -1)) recon_val = GMM( pred_in_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((x_shape[0], x_shape[1])) recon_val.name = 'gmm_out_val' recon_term_val = recon_val.sum(axis=0).mean() recon_term_val.name = 'recon_term_val' model.inputs = [x, mask, y, y_mask, scheduleSamplingMask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) header = "epoch,log,kl,nll_upper_bound,mse,mae\n" extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch, save_path, header), Monitoring( freq=monitoring_freq, #ddout=[nll_upper_bound, recon_term, kl_term, mse, mae, prediction], indexSep=5, instancesPlot=instancesPlot, #{0:[4,20],2:[5,10]},#, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] """params = OrderedDict() for node in mainloop.model.nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params)""" """ w11 = params['W_x_1__rnn'] w21 = params['W_phi_1__phi_sig'] w31 = params['W_theta_1__coeff'] print("Pickle W matricies:") print("W1:",w11.get_value()) print("W2:",w21.get_value()) print("W3:",w31.get_value())""" """mainloop.restore( name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, #cost=nll_upper_bound, #outputs=[recon_term, kl_term, nll_upper_bound, mse, mae], n_steps = n_steps, extension=extension, #lr_iterations=lr_iterations, k_speedOfconvergence=kSchedSamp )""" data = Iterator(test_data, batch_size) test_fn = theano.function( inputs=[x, y], #[x, y], #givens={x:Xtest}, #on_unused_input='ignore', #z=( ,200,1) allow_input_downcast=True, outputs=[prediction_val, recon_term_val, mse_val, mae_val] #prediction_val, mse_val, mae_val , updates= updates_val #, allow_input_downcast=True, on_unused_input='ignore' ) testOutput = [] numBatchTest = 0 for batch in data: outputGeneration = test_fn(batch[0], batch[2]) #(20, 220, 1) testOutput.append(outputGeneration[1:]) plt.figure(4) plt.plot(np.transpose(outputGeneration[0], [1, 0, 2])[4]) plt.plot(np.transpose(batch[2], [1, 0, 2])[4]) plt.savefig( save_path + "/vrnn_dis_generated{}_RealAndPred_0-4".format(numBatchTest)) plt.clf() plt.figure(4) plt.plot(np.transpose(batch[0], [1, 0, 2])[4]) plt.savefig(save_path + "/vrnn_dis_generated{}_Realagg_0-4".format(numBatchTest)) plt.clf() numBatchTest += 1 testOutput = np.asarray(testOutput) print(testOutput.shape) recon_test = testOutput[:, 0].mean() mse_test = testOutput[:, 1].mean() mae_test = testOutput[:, 2].mean() #mseUnNorm_test = testOutput[:, 3].mean() #maeUnNorm_test = testOutput[:, 4].mean() fLog = open(save_path + '/output.csv', 'w') #fLog.write(str(lr_iterations)+"\n") fLog.write(str(windows) + "\n") fLog.write("logTest,mseTest,maeTest, mseTestUnNorm, maeTestUnNorm\n") fLog.write("{},{},{}\n".format(recon_test, mse_test, mae_test)) fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{},{}\n".format(q_z_dim, p_z_dim, p_x_dim, x2s_dim, y2s_dim, z2s_dim)) header = "epoch,log,kl,mse,mae\n" fLog.write(header) for i, item in enumerate(mainloop.trainlog.monitor['recon_term']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{:d},{:.2f},{:.2f},{:.3f},{:.3f}\n".format(f, a, b, d, e))
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'dp_dis1-nosch_%d' % trial channel_name = 'mae' data_path = args['data_path'] save_path = args['save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps# int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) typeLoad = int(args['typeLoad']) debug = int(args['debug']) kSchedSamp = int(args['kSchedSamp']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 150 p_z_dim = 150 p_x_dim = 150#250 x2s_dim = 100#250 y2s_dim = 100 z2s_dim = 100#150 target_dim = k#x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_dataport(data_path, windows, appliances,numApps=flgAgg, period=period, n_steps= n_steps, stride_train = stride_train, stride_test = stride_test, trainPer=0.6, valPer=0.2, testPer=0.2, typeLoad=typeLoad, flgAggSumScaled = 1, flgFilterZeros = 1) print(reader.stdTrain, reader.meanTrain) instancesPlot = {0:[4], 2:[5]} #for now use hard coded instancesPlot for kelly sampling train_data = Dataport(name='train', prep='normalize', cond=True,# False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = Dataport(name='valid', prep='normalize', cond=True,# False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels = yval) test_data = Dataport(name='valid', prep='normalize', cond=True,# False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xtest, labels = ytest) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y , y_mask = train_data.theano_vars() scheduleSamplingMask = T.fvector('schedMask') x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp pickelModel = '/home/gissella/Documents/Research/Disaggregation/PecanStreet-dataport/VRNN_theano_version/output/gmmAE/18-05-30_16-27_app6/dp_dis1-sch_1_best.pkl' fmodel = open(pickelModel, 'rb') mainloop = cPickle.load(fmodel) fmodel.close() #define layers rnn = mainloop.model.nodes[0] x_1 = mainloop.model.nodes[1] y_1 = mainloop.model.nodes[2] z_1 = mainloop.model.nodes[3] phi_1 = mainloop.model.nodes[4] phi_mu = mainloop.model.nodes[5] phi_sig = mainloop.model.nodes[6] prior_1 = mainloop.model.nodes[7] prior_mu = mainloop.model.nodes[8] prior_sig = mainloop.model.nodes[9] theta_1 = mainloop.model.nodes[10] theta_mu = mainloop.model.nodes[11] theta_sig = mainloop.model.nodes[12] coeff = mainloop.model.nodes[13] nodes = [rnn, x_1, y_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff]#, corr, binary params = mainloop.model.params """params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params)""" s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) y_1_temp = y_1.fprop([y], params) def inner_fn_val(x_t, s_tm1): prior_1_t = prior_1.fprop([x_t,s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) pred_1_t = y_1.fprop([pred_t], params) s_t = rnn.fprop([[x_t, z_1_t, pred_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, prior_mu_t, prior_sig_t, theta_mu_t, theta_sig_t, coeff_t, pred_t#, y_pred #corr_temp, binary_temp ((s_temp_val, prior_mu_temp_val, prior_sig_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val, prediction_val), updates_val) =\ theano.scan(fn=inner_fn_val, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v s_temp_val = concatenate([s_0[None, :, :], s_temp_val[:-1]], axis=0) x_shape = x.shape ######################## TEST (GENERATION) TIME prediction_val.name = 'generated__'+str(flgAgg) mse_val = T.mean((prediction_val - y)**2) # As axis = None is calculated for all mae_val = T.mean( T.abs_(prediction_val - y) ) mse_val.name = 'mse_val' mae_val.name = 'mae_val' pred_in_val = y.reshape((y.shape[0]*y.shape[1],-1)) theta_mu_in_val = theta_mu_temp_val.reshape((x_shape[0]*x_shape[1], -1)) theta_sig_in_val = theta_sig_temp_val.reshape((x_shape[0]*x_shape[1], -1)) coeff_in_val = coeff_temp_val.reshape((x_shape[0]*x_shape[1], -1)) recon_val = GMM(pred_in_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val)# BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((x_shape[0], x_shape[1])) recon_val.name = 'gmm_out_val' recon_term_val= recon_val.sum(axis=0).mean() recon_term_val.name = 'recon_term_val' model.inputs = [x, mask, y, y_mask, scheduleSamplingMask] model.params = params model.nodes = nodes data=Iterator(test_data, batch_size) test_fn = theano.function(inputs=[x, y],#[x, y], allow_input_downcast=True, outputs=[prediction_val, recon_term_val, mse_val, mae_val]#prediction_val, mse_val, mae_val ,updates=updates_val#, allow_input_downcast=True, on_unused_input='ignore' ) testOutput = [] numBatchTest = 0 for batch in data: outputGeneration = test_fn(batch[0], batch[2])#(20, 220, 1) testOutput.append(outputGeneration[1:]) # outputGeneration[0].shape #(20, 220, 40) #if (numBatchTest<5): ''' plt.figure(1) plt.plot(np.transpose(outputGeneration[0],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_z_0-4".format(numBatchTest)) plt.clf() plt.figure(2) plt.plot(np.transpose(outputGeneration[1],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_s_0-4".format(numBatchTest)) plt.clf() plt.figure(3) plt.plot(np.transpose(outputGeneration[2],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_theta_0-4".format(numBatchTest)) plt.clf() ''' plt.figure(4) plt.plot(np.transpose(outputGeneration[0],[1,0,2])[4]) plt.plot(np.transpose(batch[2],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_RealAndPred_0-4".format(numBatchTest)) plt.clf() plt.figure(4) plt.plot(np.transpose(batch[0],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_Realagg_0-4".format(numBatchTest)) plt.clf() numBatchTest+=1 testOutput = np.asarray(testOutput) print(testOutput.shape) recon_test = testOutput[:, 0].mean() mse_test = testOutput[:, 1].mean() mae_test = testOutput[:, 2].mean() #mseUnNorm_test = testOutput[:, 3].mean() #maeUnNorm_test = testOutput[:, 4].mean() fLog = open(save_path+'/output.csv', 'w') fLog.write(str(lr_iterations)+"\n") fLog.write(str(windows)+"\n") fLog.write("logTest,mseTest,maeTest, mseTestUnNorm, maeTestUnNorm\n") fLog.write("{},{},{}\n".format(recon_test,mse_test,mae_test)) fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{},{}\n".format(q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim)) header = "epoch,log,kl,mse,mae\n" fLog.write(header) for i , item in enumerate(mainloop.trainlog.monitor['recon_term']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{:d},{:.2f},{:.2f},{:.3f},{:.3f}\n".format(f,a,b,d,e))
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'dp_dis1-nosch_%d' % trial channel_name = 'mae' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) typeLoad = int(args['typeLoad']) debug = int(args['debug']) kSchedSamp = int(args['kSchedSamp']) typeActivFunc = args['typeActivFunc'] print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 150 p_z_dim = 150 p_x_dim = 150 #250 x2s_dim = 100 #250 y2s_dim = 100 z2s_dim = 100 #150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_dataport( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, trainPer=0.6, valPer=0.2, testPer=0.2, typeLoad=typeLoad, flgAggSumScaled=1, flgFilterZeros=1) print(reader.stdTrain, reader.meanTrain) instancesPlot = { 0: [4], 2: [5] } #for now use hard coded instancesPlot for kelly sampling train_data = Dataport( name='train', prep='normalize', cond=True, # False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels=yval) test_data = Dataport( name='valid', prep='normalize', cond=True, # False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xtest, labels=ytest) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() scheduleSamplingMask = T.fvector('schedMask') x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) y_1 = FullyConnectedLayer(name='y_1', parent=['y_t'], parent_dim=[y_dim], nout=y2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1', 'z_1', 'y_1'], parent_dim=[x2s_dim, z2s_dim, y_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1', 'y_1'], parent_dim=[x2s_dim, rnn_dim, y2s_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['x_1', 's_tm1'], parent_dim=[x2s_dim, rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit=typeActivFunc, init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[p_x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) nodes = [ rnn, x_1, y_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) y_1_temp = y_1.fprop([y], params) def inner_fn_train(x_t, y_t, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1, y_t], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([x_t, s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) s_t = rnn.fprop([[x_t, z_1_t, y_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, theta_mu_t, theta_sig_t, coeff_t, pred #, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp, theta_mu_temp, theta_sig_temp, coeff_temp, prediction), updates) =\ theano.scan(fn=inner_fn_train, sequences=[x_1_temp, y_1_temp], outputs_info=[s_0, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v #s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0)# seems like this is for creating an additional dimension to s_0 theta_mu_temp.name = 'theta_mu_temp' theta_sig_temp.name = 'theta_sig_temp' coeff_temp.name = 'coeff' if (flgAgg == -1): prediction.name = 'x_reconstructed' mse = T.mean((prediction - x)**2) # CHECK RESHAPE with an assertion mae = T.mean(T.abs(prediction - x)) mse.name = 'mse' pred_in = x.reshape((x_shape[0] * x_shape[1], -1)) else: prediction.name = 'pred_' + str(flgAgg) mse = T.mean( (prediction - y)**2) # As axis = None is calculated for all mae = T.mean(T.abs_(prediction - y)) mse.name = 'mse' mae.name = 'mae' pred_in = y.reshape((y.shape[0] * y.shape[1], -1)) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0] * x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) recon = GMM( pred_in, theta_mu_in, theta_sig_in, coeff_in ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon.name = 'gmm_out' recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' nll_upper_bound = recon_term + kl_term #+ mse if (flgMSE): nll_upper_bound = nll_upper_bound + mse nll_upper_bound.name = 'nll_upper_bound' model.inputs = [x, mask, y, y_mask, scheduleSamplingMask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) header = "epoch,log,kl,nll_upper_bound,mse,mae\n" extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch, save_path, header), Monitoring( freq=monitoring_freq, ddout=[ nll_upper_bound, recon_term, kl_term, mse, mae, theta_mu_temp, prediction ], indexSep=5, instancesPlot=instancesPlot, #{0:[4,20],2:[5,10]},#, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = {0: lr} mainloop = Training( name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[recon_term, kl_term, nll_upper_bound, mse, mae], n_steps=n_steps, extension=extension, lr_iterations=lr_iterations, k_speedOfconvergence=kSchedSamp) mainloop.run() fLog = open(save_path + '/output.csv', 'w') fLog.write(str(lr_iterations) + "\n") fLog.write(str(windows) + "\n") fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{},{}\n".format(q_z_dim, p_z_dim, p_x_dim, x2s_dim, y2s_dim, z2s_dim)) header = "epoch,log,kl,mse,mae\n" fLog.write(header) for i, item in enumerate(mainloop.trainlog.monitor['recon_term']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{:d},{:.2f},{:.2f},{:.3f},{:.3f}\n".format(f, a, b, d, e))
theano.scan(fn=inner_fn, sequences=[x], outputs_info=[coder.get_init_state(), None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v reshaped_x = x.reshape((x.shape[0]*x.shape[1], -1)) reshaped_theta_mu = theta_mu_t.reshape((theta_mu_t.shape[0]*theta_mu_t.shape[1], -1)) reshaped_theta_sig = theta_sig_t.reshape((theta_sig_t.shape[0]*theta_sig_t.shape[1], -1)) reshaped_coeff = coeff_t.reshape((coeff_t.shape[0]*coeff_t.shape[1], -1)) reshaped_mask = mask.flatten() kl_term = kl_t.reshape((kl_t.shape[0]*kl_t.shape[1], -1)) recon_term = GMM(reshaped_x, reshaped_theta_mu, reshaped_theta_sig, reshaped_coeff) # Apply mask kl_term = kl_term[reshaped_mask.nonzero()].mean() recon_term = recon_term[reshaped_mask.nonzero()].mean() cost = recon_term + kl_term cost.name = 'cost' recon_term.name = 'recon_term' kl_term.name = 'kl_term' max_x = x.max() mean_x = x.mean() min_x = x.min() max_x.name = 'max_x' mean_x.name = 'mean_x' min_x.name = 'min_x'
def main(args): theano.optimizer = 'fast_compile' theano.config.exception_verbosity = 'high' trial = int(args['trial']) pkl_name = 'vrnn_gmm_%d' % trial channel_name = 'mse_val' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) genCase = int(args['genCase']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps # int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) debug = int(args['debug']) num_sequences_per_batch = int(args['numSequences']) #based on appliance loadParam = args['loadAsKelly'] target_inclusion_prob = float(args['target_inclusion_prob']) loadAsKelly = True if (loadParam == 'N' or loadParam == 'n' or loadParam == 'no' or loadParam == 'NO' or loadParam == 'No'): loadAsKelly = False print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 60 #150 p_z_dim = 60 #150 p_x_dim = 30 #250 x2s_dim = 40 #250 z2s_dim = 40 #150 target_dim = k #x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, reader = fetch_ukdale( data_path, windows, appliances, numApps=flgAgg, period=period, n_steps=n_steps, stride_train=stride_train, stride_test=stride_test, flgAggSumScaled=1, flgFilterZeros=1, isKelly=loadAsKelly, seq_per_batch=num_sequences_per_batch, target_inclusion_prob=target_inclusion_prob) instancesPlot = { 0: [10, 20], 2: [20, 30] } #for now use hard coded instancesPlot for kelly sampling if (not loadAsKelly): instancesPlot = reader.build_dict_instances_plot( listDates, batch_size, Xval.shape[0]) ############# We switch x with y train_data = UKdale( name='train', prep='normalize', cond=False, #path=data_path, validTime=0, inputX=ytrain, labels=Xtrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = UKdale( name='valid', prep='normalize', cond=False, #path=data_path, X_mean=X_mean, X_std=X_std, validTime=1, inputX=yval, labels=Xval) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask = train_data.theano_vars() #valTime = train_data.theano_valTime_vars() if (genCase == 1): inputX = x[:-1, :] targetX = x[1:, :] n_steps = n_steps - 1 else: inputX = x targetX = x inputX.name = 'x_original' if debug: inputX.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1', 'z_1'], parent_dim=[x2s_dim, z2s_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1'], parent_dim=[x2s_dim, rnn_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[p_x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) nodes = [ rnn, x_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) #Creates matrices params = init_tparams(params) #Make the parameters theano.shared s_0_tr = rnn.get_init_state(batch_size) s_0_val = T.zeros((batch_size, 2 * rnn_dim), dtype=theano.config.floatX) s_0_val = T.unbroadcast( s_0_val, *range(s_0_val.ndim) ) #[0,1] this is to raise an error if length of dimensions are not 1 #x_1_temp = x_1.fprop([x], params) def inner_val_fn(s_tm1): ''' phi_1_t = phi_1.fprop([x_t, s_tm1], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) ''' prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) x_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) x_1_t = x_1.fprop([x_t], params) s_t = rnn.fprop([[x_1_t, z_1_t], [s_tm1]], params) return s_t, x_t, z_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t # prior_mu_temp_val, prior_sig_temp_val ((s_temp_val, prediction_val, z_t_temp_val, theta_1_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val), updates_val) =\ theano.scan(fn=inner_val_fn , n_steps=n_steps, #already 1 subtracted if doing next step outputs_info=[s_0_val, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v def inner_train_fn(x_t, s_tm1): x_1_t = x_1.fprop([x_t], params) phi_1_t = phi_1.fprop([x_1_t, s_tm1], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) s_t = rnn.fprop([[x_1_t, z_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t, pred #, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp,z_t_temp, z_1_temp, theta_1_temp, theta_mu_temp, theta_sig_temp, coeff_temp, prediction), updates) =\ theano.scan(fn=inner_train_fn, sequences=[inputX],#[x_1_temp], outputs_info=[s_0_tr, None, None, None, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v ######### TRAINING GRAPH ######### s_temp = concatenate( [s_0_tr[None, :, :], s_temp[:-1]], axis=0 ) # seems like this is for creating an additional dimension to s_0 s_temp.name = 'h_1' #gisse z_1_temp.name = 'z_1' #gisse z_t_temp.name = 'z' theta_mu_temp.name = 'theta_mu_temp' theta_sig_temp.name = 'theta_sig_temp' coeff_temp.name = 'coeff' prediction.name = 'pred_' + str(flgAgg) mse = T.mean( (prediction - targetX)**2) # As axis = None is calculated for all mae = T.mean(T.abs_(prediction - targetX)) mse.name = 'mse' mae.name = 'mae' x_in = inputX.reshape((batch_size * n_steps, -1)) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) target_shape = x[:, 1:].shape theta_mu_in = theta_mu_temp.reshape((batch_size * n_steps, -1)) theta_sig_in = theta_sig_temp.reshape((batch_size * n_steps, -1)) coeff_in = coeff_temp.reshape((batch_size * n_steps, -1)) recon = GMM( x_in, theta_mu_in, theta_sig_in, coeff_in ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((batch_size, n_steps)) recon.name = 'gmm_out' recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' nll_upper_bound = recon_term + kl_term #+ mse if (flgMSE): nll_upper_bound = nll_upper_bound + mse nll_upper_bound.name = 'nll_upper_bound' ######### TESTING GRAPH ######### s_temp_val = concatenate( [s_0_val[None, :, :], s_temp_val[:-1]], axis=0 ) # seems like this is for creating an additional dimension to s_0 s_temp_val.name = 'h_1_val' #gisse #z_1_temp_val.name = 'z_1_val'#gisse z_t_temp_val.name = 'z_val' theta_mu_temp_val.name = 'theta_mu_temp_val' theta_sig_temp_val.name = 'theta_sig_temp_val' coeff_temp_val.name = 'coeff_val' prediction_val.name = 'generated_' + str(flgAgg) mse_val = T.mean( (prediction_val - targetX)**2) # As axis = None is calculated for all mae_val = T.mean(T.abs_(prediction_val - targetX)) mse_val.name = 'mse_val' mae_val.name = 'mae_val' x_in_val = inputX.reshape((batch_size * n_steps, -1)) # No sense in calculate distance to a distribution because we are not calculating phi #kl_temp_val = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) theta_mu_in_val = theta_mu_temp_val.reshape((batch_size * n_steps, -1)) theta_sig_in_val = theta_sig_temp_val.reshape((batch_size * n_steps, -1)) coeff_in_val = coeff_temp_val.reshape((batch_size * n_steps, -1)) recon_val = GMM( x_in_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((batch_size, n_steps)) recon_val.name = 'gmm_out_val' recon_term_val = recon_val.sum(axis=0).mean() recon_term_val.name = 'recon_term_val' ###################################### model.inputs = [x, mask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) header = "epoch,log,kl,nll,mse,mae\n" extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch, save_path, header), Monitoring( freq=monitoring_freq, ddout=[ recon_term_val, mse_val, mae_val, prediction_val, theta_mu_temp_val ], indexSep=3, indexDDoutPlot=[(0, prediction_val)], instancesPlot=instancesPlot, #, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = {0: lr} mainloop = Training( name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[recon_term, kl_term, nll_upper_bound, mse, mae], extension=extension, lr_iterations=lr_iterations) mainloop.run() fLog = open(save_path + '/output.csv', 'w') fLog.write(str(lr_iterations) + "\n") fLog.write(str(windows) + "\n") fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{}\n".format(q_z_dim, p_z_dim, p_x_dim, x2s_dim, z2s_dim)) header = "epoch,log,mse,mae\n" fLog.write(header) for i, item in enumerate(mainloop.trainlog.monitor['recon_term_val']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term_val'][i] d = mainloop.trainlog.monitor['mse_val'][i] e = mainloop.trainlog.monitor['mae_val'][i] fLog.write("{},{},{},{}\n".format(f, a, d, e))
def main(args): #theano.optimizer='fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'vrnn_gmm_%d' % trial channel_name = 'nll_upper_bound' data_path = args['data_path'] save_path = args['save_path'] #+'/gmm/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") flgMSE = int(args['flgMSE']) genCase = int(args['genCase']) period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = n_steps#int(args['stride_test']) monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) flgAgg = int(args['flgAgg']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) debug = int(args['debug']) num_sequences_per_batch = int(args['numSequences']) #based on appliance typeLoad = int(args['typeLoad']) target_inclusion_prob = float(args['target_inclusion_prob']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 150 p_z_dim = 150 p_x_dim = 200 x2s_dim = 100 y2s_dim = 100 z2s_dim = 100 target_dim = k#x_dim #(x_dim-1)*k model = Model() Xtrain, ytrain, Xval, yval, Xtest, ytest, reader = fetch_ukdale(data_path, windows, appliances,numApps=flgAgg, period=period, n_steps= n_steps, stride_train = stride_train, stride_test = stride_test, typeLoad= typeLoad, flgAggSumScaled = 1, flgFilterZeros = 1, seq_per_batch=num_sequences_per_batch, target_inclusion_prob=target_inclusion_prob) instancesPlot = {0:[4,20], 2:[5,10]} #for now use hard coded instancesPlot for kelly sampling if(typeLoad==0): instancesPlot = reader.build_dict_instances_plot(listDates, batch_size, Xval.shape[0]) train_data = UKdale(name='train', prep='normalize', cond=True,# False #path=data_path, inputX=Xtrain, labels=ytrain) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = UKdale(name='valid', prep='normalize', cond=True,# False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xval, labels = yval) test_data = UKdale(name='valid', prep='normalize', cond=True,# False #path=data_path, X_mean=X_mean, X_std=X_std, inputX=Xtest, labels = ytest) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y , y_mask = train_data.theano_vars() if (genCase ==1): inputX = x[:-1,:] targetX = x[1:,:] n_steps = n_steps-1 else: inputX = x targetX = x x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) y_1 = FullyConnectedLayer(name='y_1', parent=['y_t'], parent_dim=[y_dim], nout=y2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1', 'z_1','y_1'], parent_dim=[x2s_dim, z2s_dim, y_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1','y_1'], parent_dim=[x2s_dim, rnn_dim,y2s_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer(name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', init_W=init_W, init_b=init_b) corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[p_x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) nodes = [rnn, x_1, y_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff]#, corr, binary params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) y_1_temp = y_1.fprop([y], params) def inner_fn_val(x_t, s_tm1): prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(prior_mu_t, prior_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) pred_t = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) pred_1_t = y_1.fprop([pred_t], params) s_t = rnn.fprop([[x_t, z_1_t, pred_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t, pred_t#, y_pred #corr_temp, binary_temp ((s_temp_val, prior_mu_temp_val, prior_sig_temp_val, z_t_temp_val, z_1_temp_val, theta_1_temp_val, theta_mu_temp_val, theta_sig_temp_val, coeff_temp_val, prediction_val), updates_val) =\ theano.scan(fn=inner_fn_val, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None, None, None, None]) for k, v in updates_val.iteritems(): k.default_update = v s_temp_val = concatenate([s_0[None, :, :], s_temp_val[:-1]], axis=0) def inner_fn_train(x_t, y_t, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1,y_t], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample(phi_mu_t, phi_sig_t) z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) pred = GMM_sample(theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) s_t = rnn.fprop([[x_t, z_1_t, y_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t, pred#, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp,z_t_temp, z_1_temp, theta_1_temp, theta_mu_temp, theta_sig_temp, coeff_temp, prediction), updates) =\ theano.scan(fn=inner_fn_train, sequences=[x_1_temp, y_1_temp], outputs_info=[s_0, None, None, None, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v s_temp = concatenate([s_0[None, :, :], s_temp[:-1]], axis=0)# seems like this is for creating an additional dimension to s_0 s_temp.name = 'h_1'#gisse z_1_temp.name = 'z_1'#gisse z_t_temp.name = 'z' theta_mu_temp.name = 'theta_mu_temp' theta_sig_temp.name = 'theta_sig_temp' coeff_temp.name = 'coeff' if (flgAgg == -1 ): prediction.name = 'x_reconstructed' mse = T.mean((prediction - x)**2) # CHECK RESHAPE with an assertion mae = T.mean( T.abs(prediction - x) ) mse.name = 'mse' pred_in = x.reshape((x_shape[0]*x_shape[1], -1)) else: prediction.name = 'pred_'+str(flgAgg) mse = T.mean((prediction - y)**2) # As axis = None is calculated for all mae = T.mean( T.abs_(prediction - y) ) mse.name = 'mse' mae.name = 'mae' pred_in = y.reshape((y.shape[0]*y.shape[1],-1)) kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape theta_mu_in = theta_mu_temp.reshape((x_shape[0]*x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0]*x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0]*x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) recon = GMM(pred_in, theta_mu_in, theta_sig_in, coeff_in)# BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon.name = 'gmm_out' recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' nll_upper_bound = recon_term + kl_term #+ mse if (flgMSE): nll_upper_bound = nll_upper_bound + mse nll_upper_bound.name = 'nll_upper_bound' ######################## TEST (GENERATION) TIME prediction_val.name = 'generated__'+str(flgAgg) mse_val = T.mean((prediction_val - y)**2) # As axis = None is calculated for all mae_val = T.mean( T.abs_(prediction_val - y) ) mse_val.name = 'mse_val' mae_val.name = 'mae_val' pred_in_val = y.reshape((y.shape[0]*y.shape[1],-1)) theta_mu_in_val = theta_mu_temp_val.reshape((x_shape[0]*x_shape[1], -1)) theta_sig_in_val = theta_sig_temp_val.reshape((x_shape[0]*x_shape[1], -1)) coeff_in_val = coeff_temp_val.reshape((x_shape[0]*x_shape[1], -1)) recon_val = GMM(pred_in_val, theta_mu_in_val, theta_sig_in_val, coeff_in_val)# BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon_val = recon_val.reshape((x_shape[0], x_shape[1])) recon_val.name = 'gmm_out_val' recon_term_val= recon_val.sum(axis=0).mean() recon_term_val.name = 'recon_term_val' model.inputs = [x, mask, y, y_mask] model.params = params model.nodes = nodes optimizer = Adam( lr=lr ) header = "epoch,log,kl,nll_upper_bound,mse,mae\n" extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch, save_path, header), Monitoring(freq=monitoring_freq, ddout=[nll_upper_bound, recon_term, kl_term, mse, mae, theta_mu_temp,prediction], indexSep=5, instancesPlot = instancesPlot, #{0:[4,20],2:[5,10]},#, 80,150 data=[Iterator(valid_data, batch_size)], savedFolder = save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] lr_iterations = {0:lr, 75:(lr/10), 150:(lr/100)} mainloop = Training( name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[recon_term, kl_term, nll_upper_bound, mse, mae], extension=extension, lr_iterations=lr_iterations ) mainloop.run() z_t_temp_val.name='z_temp_val' s_temp_val.name='s_temp_val' theta_mu_temp_val.name='mu_temp' data=Iterator(test_data, batch_size) test_fn = theano.function(inputs=[x, y],#[x, y], #givens={x:Xtest}, #on_unused_input='ignore', #z=( ,200,1) allow_input_downcast=True, outputs=[z_t_temp_val, s_temp_val, theta_mu_temp_val, prediction_val, recon_term_val, mse_val, mae_val]#prediction_val, mse_val, mae_val ,updates=updates_val#, allow_input_downcast=True, on_unused_input='ignore' ) testOutput = [] numBatchTest = 0 for batch in data: outputGeneration = test_fn(batch[0], batch[2]) testOutput.append(outputGeneration[4:]) #{0:[4,20], 2:[5,10]} #if (numBatchTest==0): ''' plt.figure(1) plt.plot(np.transpose(outputGeneration[0],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_z_0-4".format(numBatchTest)) plt.clf() plt.figure(2) plt.plot(np.transpose(outputGeneration[1],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_s_0-4".format(numBatchTest)) plt.clf() plt.figure(3) plt.plot(np.transpose(outputGeneration[2],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_theta_0-4".format(numBatchTest)) plt.clf() ''' plt.figure(4) plt.plot(np.transpose(outputGeneration[3],[1,0,2])[4]) plt.plot(np.transpose(batch[2],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_RealAndPred_0-4".format(numBatchTest)) plt.clf() plt.figure(4) plt.plot(np.transpose(batch[0],[1,0,2])[4]) plt.savefig(save_path+"/vrnn_dis_generated{}_Realagg_0-4".format(numBatchTest)) plt.clf() numBatchTest+=1 testOutput = np.asarray(testOutput) print(testOutput.shape) recon_test = this_mean = testOutput[:, 0].mean() mse_test = this_mean = testOutput[:, 1].mean() mae_test = this_mean = testOutput[:, 2].mean() fLog = open(save_path+'/output.csv', 'w') fLog.write(str(lr_iterations)+"\n") fLog.write(str(windows)+"\n") fLog.write("logTest,mseTest,maeTest\n") fLog.write("{},{},{}\n".format(recon_test,mse_test,mae_test)) fLog.write("q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim\n") fLog.write("{},{},{},{},{},{}\n".format(q_z_dim,p_z_dim,p_x_dim,x2s_dim,y2s_dim,z2s_dim)) header = "epoch,log,kl,mse,mae\n" fLog.write(header) for i , item in enumerate(mainloop.trainlog.monitor['recon_term']): f = mainloop.trainlog.monitor['epoch'][i] a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] d = mainloop.trainlog.monitor['mse'][i] e = mainloop.trainlog.monitor['mae'][i] fLog.write("{:d},{:.2f},{:.2f},{:.3f},{:.3f}\n".format(f,a,b,d,e))
def main(args): theano.optimizer = 'fast_compile' #theano.config.exception_verbosity='high' trial = int(args['trial']) pkl_name = 'vrnn_gmm_%d' % trial channel_name = 'valid_nll_upper_bound' data_path = args['data_path'] save_path = args[ 'save_path'] #+'/aggVSdisag_distribEach/'+datetime.datetime.now().strftime("%y-%m-%d_%H-%M") period = int(args['period']) n_steps = int(args['n_steps']) stride_train = int(args['stride_train']) stride_test = int(args['stride_test']) # monitoring is a value that represents how many batches from training have to be seen to measure the validation set monitoring_freq = int(args['monitoring_freq']) epoch = int(args['epoch']) batch_size = int(args['batch_size']) x_dim = int(args['x_dim']) y_dim = int(args['y_dim']) z_dim = int(args['z_dim']) rnn_dim = int(args['rnn_dim']) k = int(args['num_k']) #a mixture of K Gaussian functions lr = float(args['lr']) debug = int(args['debug']) print "trial no. %d" % trial print "batch size %d" % batch_size print "learning rate %f" % lr print "saving pkl file '%s'" % pkl_name print "to the save path '%s'" % save_path q_z_dim = 8 #150 p_z_dim = 8 #150 p_x_dim = 7 #250 x2s_dim = 7 #250 z2s_dim = 8 #150 target_dim = y_dim * k #(x_dim-1)*k model = Model() train_data = UKdale( name='train', prep='normalize', cond=True, # False path=data_path, period=period, n_steps=n_steps, x_dim=x_dim, stride_train=stride_train, stride_test=stride_test) X_mean = train_data.X_mean X_std = train_data.X_std valid_data = UKdale( name='valid', prep='normalize', cond=True, # False path=data_path, X_mean=X_mean, X_std=X_std, period=period, n_steps=n_steps, x_dim=x_dim, stride_train=stride_train, stride_test=stride_test) init_W = InitCell('rand') init_U = InitCell('ortho') init_b = InitCell('zeros') init_b_sig = InitCell('const', mean=0.6) x, mask, y, y_mask = train_data.theano_vars() x.name = 'x_original' if debug: x.tag.test_value = np.zeros((15, batch_size, x_dim), dtype=np.float32) temp = np.ones((15, batch_size), dtype=np.float32) temp[:, -2:] = 0. mask.tag.test_value = temp x_1 = FullyConnectedLayer(name='x_1', parent=['x_t'], parent_dim=[x_dim], nout=x2s_dim, unit='relu', init_W=init_W, init_b=init_b) z_1 = FullyConnectedLayer(name='z_1', parent=['z_t'], parent_dim=[z_dim], nout=z2s_dim, unit='relu', init_W=init_W, init_b=init_b) rnn = LSTM(name='rnn', parent=['x_1', 'z_1'], parent_dim=[x2s_dim, z2s_dim], nout=rnn_dim, unit='tanh', init_W=init_W, init_U=init_U, init_b=init_b) phi_1 = FullyConnectedLayer(name='phi_1', parent=['x_1', 's_tm1'], parent_dim=[x2s_dim, rnn_dim], nout=q_z_dim, unit='relu', init_W=init_W, init_b=init_b) phi_mu = FullyConnectedLayer(name='phi_mu', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) phi_sig = FullyConnectedLayer(name='phi_sig', parent=['phi_1'], parent_dim=[q_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) prior_1 = FullyConnectedLayer(name='prior_1', parent=['s_tm1'], parent_dim=[rnn_dim], nout=p_z_dim, unit='relu', init_W=init_W, init_b=init_b) prior_mu = FullyConnectedLayer(name='prior_mu', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='linear', init_W=init_W, init_b=init_b) prior_sig = FullyConnectedLayer(name='prior_sig', parent=['prior_1'], parent_dim=[p_z_dim], nout=z_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) theta_1 = FullyConnectedLayer(name='theta_1', parent=['z_1', 's_tm1'], parent_dim=[z2s_dim, rnn_dim], nout=p_x_dim, unit='relu', init_W=init_W, init_b=init_b) theta_mu = FullyConnectedLayer(name='theta_mu', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='linear', init_W=init_W, init_b=init_b) theta_sig = FullyConnectedLayer(name='theta_sig', parent=['theta_1'], parent_dim=[p_x_dim], nout=target_dim, unit='softplus', cons=1e-4, init_W=init_W, init_b=init_b_sig) coeff = FullyConnectedLayer( name='coeff', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='softmax', #to ensure that the sum adds to one init_W=init_W, init_b=init_b) corr = FullyConnectedLayer(name='corr', parent=['theta_1'], parent_dim=[p_x_dim], nout=k, unit='tanh', init_W=init_W, init_b=init_b) binary = FullyConnectedLayer(name='binary', parent=['theta_1'], parent_dim=[p_x_dim], nout=1, unit='sigmoid', init_W=init_W, init_b=init_b) nodes = [ rnn, x_1, z_1, #dissag_pred, phi_1, phi_mu, phi_sig, prior_1, prior_mu, prior_sig, theta_1, theta_mu, theta_sig, coeff ] #, corr, binary params = OrderedDict() for node in nodes: if node.initialize() is not None: params.update(node.initialize()) params = init_tparams(params) s_0 = rnn.get_init_state(batch_size) x_1_temp = x_1.fprop([x], params) def inner_fn(x_t, s_tm1): phi_1_t = phi_1.fprop([x_t, s_tm1], params) phi_mu_t = phi_mu.fprop([phi_1_t], params) phi_sig_t = phi_sig.fprop([phi_1_t], params) prior_1_t = prior_1.fprop([s_tm1], params) prior_mu_t = prior_mu.fprop([prior_1_t], params) prior_sig_t = prior_sig.fprop([prior_1_t], params) z_t = Gaussian_sample( phi_mu_t, phi_sig_t ) #in the original code it is gaussian. GMM is for the generation z_1_t = z_1.fprop([z_t], params) theta_1_t = theta_1.fprop([z_1_t, s_tm1], params) theta_mu_t = theta_mu.fprop([theta_1_t], params) theta_sig_t = theta_sig.fprop([theta_1_t], params) coeff_t = coeff.fprop([theta_1_t], params) #corr_t = corr.fprop([theta_1_t], params) #binary_t = binary.fprop([theta_1_t], params) # I was missing this reshape that is done before BiGMM in the original code ''' theta_mu_in = theta_mu_t.reshape((x_t[0]*x_t[1], -1)) theta_sig_in = theta_sig_t.reshape((x_t[0]*x_t[1], -1)) coeff_in = coeff_t.reshape((x_t[0]*x_t[1], -1)) ''' y_pred = GMM_sampleY( theta_mu_t, theta_sig_t, coeff_t) #Gaussian_sample(theta_mu_t, theta_sig_t) s_t = rnn.fprop([[x_t, z_1_t], [s_tm1]], params) #y_pred = dissag_pred.fprop([s_t], params) return s_t, phi_mu_t, phi_sig_t, prior_mu_t, prior_sig_t, z_t, z_1_t, theta_1_t, theta_mu_t, theta_sig_t, coeff_t, y_pred #corr_temp, binary_temp ((s_temp, phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp,z_t_temp, z_1_temp, theta_1_temp, theta_mu_temp, theta_sig_temp, coeff_temp, y_pred_temp), updates) =\ theano.scan(fn=inner_fn, sequences=[x_1_temp], outputs_info=[s_0, None, None, None, None, None, None, None, None, None, None, None]) for k, v in updates.iteritems(): k.default_update = v s_temp = concatenate( [s_0[None, :, :], s_temp[:-1]], axis=0 ) # seems like this is for creating an additional dimension to s_0 ''' theta_1_temp = theta_1.fprop([z_1_temp, s_temp], params) theta_mu_temp = theta_mu.fprop([theta_1_temp], params) theta_sig_temp = theta_sig.fprop([theta_1_temp], params) coeff_temp = coeff.fprop([theta_1_temp], params) corr_temp = corr.fprop([theta_1_temp], params) binary_temp = binary.fprop([theta_1_temp], params) ''' s_temp.name = 'h_1' #gisse z_1_temp.name = 'z_1' #gisse z_t_temp.name = 'z' theta_mu_temp.name = 'theta_mu' theta_sig_temp.name = 'theta_sig' coeff_temp.name = 'coeff' #corr_temp.name = 'corr' #binary_temp.name = 'binary' #x_pred_temp.name = 'x_reconstructed' y_pred_temp.name = 'disaggregation' #mse = T.mean((y_pred_temp - y)**2) # cause mse can be 26000 #mse.name = 'mse' kl_temp = KLGaussianGaussian(phi_mu_temp, phi_sig_temp, prior_mu_temp, prior_sig_temp) x_shape = x.shape y_shape = y.shape x_in = x.reshape((x_shape[0] * x_shape[1], -1)) y_in = y.reshape((y_shape[0] * y_shape[1], -1)) theta_mu_in = theta_mu_temp.reshape((x_shape[0] * x_shape[1], -1)) theta_sig_in = theta_sig_temp.reshape((x_shape[0] * x_shape[1], -1)) coeff_in = coeff_temp.reshape((x_shape[0] * x_shape[1], -1)) #corr_in = corr_temp.reshape((x_shape[0]*x_shape[1], -1)) #binary_in = binary_temp.reshape((x_shape[0]*x_shape[1], -1)) #print("Printing shapes") #print (y_in.shape, theta_mu_in.shape, theta_sig_in.shape, coeff_in.shape) recon = GMM( y_in, theta_mu_in, theta_sig_in, coeff_in ) # BiGMM(x_in, theta_mu_in, theta_sig_in, coeff_in, corr_in, binary_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon.name = 'gmm_out' #recon = recon * mask recon_term = recon.sum(axis=0).mean() recon_term.name = 'recon_term' #kl_temp = kl_temp * mask kl_term = kl_temp.sum(axis=0).mean() kl_term.name = 'kl_term' #nll_upper_bound_0 = recon_term + kl_term #nll_upper_bound_0.name = 'nll_upper_bound_0' nll_upper_bound = recon_term + kl_term #+ mse nll_upper_bound.name = 'nll_upper_bound' max_x = x.max() mean_x = x.mean() min_x = x.min() max_x.name = 'max_x' mean_x.name = 'mean_x' min_x.name = 'min_x' max_theta_mu = theta_mu_in.max() mean_theta_mu = theta_mu_in.mean() min_theta_mu = theta_mu_in.min() max_theta_mu.name = 'max_theta_mu' mean_theta_mu.name = 'mean_theta_mu' min_theta_mu.name = 'min_theta_mu' max_theta_sig = theta_sig_in.max() mean_theta_sig = theta_sig_in.mean() min_theta_sig = theta_sig_in.min() max_theta_sig.name = 'max_theta_sig' mean_theta_sig.name = 'mean_theta_sig' min_theta_sig.name = 'min_theta_sig' coeff_max = coeff_in.max() coeff_min = coeff_in.min() coeff_mean_max = coeff_in.mean(axis=0).max() coeff_mean_min = coeff_in.mean(axis=0).min() coeff_max.name = 'coeff_max' coeff_min.name = 'coeff_min' coeff_mean_max.name = 'coeff_mean_max' coeff_mean_min.name = 'coeff_mean_min' max_phi_sig = phi_sig_temp.max() mean_phi_sig = phi_sig_temp.mean() min_phi_sig = phi_sig_temp.min() max_phi_sig.name = 'max_phi_sig' mean_phi_sig.name = 'mean_phi_sig' min_phi_sig.name = 'min_phi_sig' max_prior_sig = prior_sig_temp.max() mean_prior_sig = prior_sig_temp.mean() min_prior_sig = prior_sig_temp.min() max_prior_sig.name = 'max_prior_sig' mean_prior_sig.name = 'mean_prior_sig' min_prior_sig.name = 'min_prior_sig' model.inputs = [x, mask, y, y_mask] model.params = params model.nodes = nodes optimizer = Adam(lr=lr) extension = [ GradientClipping(batch_size=batch_size), EpochCount(epoch), Monitoring( freq=monitoring_freq, ddout=[ nll_upper_bound, recon_term, kl_term, #mse, max_phi_sig, mean_phi_sig, min_phi_sig, max_prior_sig, mean_prior_sig, min_prior_sig, max_theta_sig, mean_theta_sig, min_theta_sig, max_x, mean_x, min_x, max_theta_mu, mean_theta_mu, min_theta_mu, coeff_max, coeff_min, coeff_mean_max, coeff_mean_min, #23 theta_mu_temp, theta_sig_temp, z_t_temp, y_pred_temp, #corr_temp, binary_temp, coeff_temp, #22 s_temp, z_1_temp ], indexSep=22, indexDDoutPlot=[(0, theta_mu_temp), (2, z_t_temp), (3, y_pred_temp) ], # adding indexes of ddout for the plotting instancesPlot=[0, 150], data=[Iterator(valid_data, batch_size)], savedFolder=save_path), Picklize(freq=monitoring_freq, path=save_path), EarlyStopping(freq=monitoring_freq, path=save_path, channel=channel_name), WeightNorm() ] mainloop = Training(name=pkl_name, data=Iterator(train_data, batch_size), model=model, optimizer=optimizer, cost=nll_upper_bound, outputs=[nll_upper_bound], extension=extension) mainloop.run() fLog = open(save_path + '/output.csv', 'w') print('Printing') print(len(mainloop.trainlog.monitor['nll_upper_bound'])) fLog.write("log,kl,mse,nll_upper_bound\n") for i, item in enumerate(mainloop.trainlog.monitor['nll_upper_bound']): a = mainloop.trainlog.monitor['recon_term'][i] b = mainloop.trainlog.monitor['kl_term'][i] #c = mainloop.trainlog.monitor['mse'][i] d = mainloop.trainlog.monitor['nll_upper_bound'] #print(a,b) fLog.write("{},{},{}\n".format(a, b, d)) fLog.close()
theta_2_in = theta_2.fprop([theta_1_in]) theta_3_in = theta_3.fprop([theta_2_in]) theta_4_in = theta_4.fprop([theta_3_in]) theta_mu_in = theta_mu.fprop([theta_4_in]) theta_sig_in = theta_sig.fprop([theta_4_in]) coeff_in = coeff.fprop([theta_4_in]) z_shape = phi_mu_t.shape phi_mu_in = phi_mu_t.reshape((z_shape[0]*z_shape[1], -1)) phi_sig_in = phi_sig_t.reshape((z_shape[0]*z_shape[1], -1)) prior_mu_in = prior_mu_t.reshape((z_shape[0]*z_shape[1], -1)) prior_sig_in = prior_sig_t.reshape((z_shape[0]*z_shape[1], -1)) kl_in = kl.fprop([phi_mu_in, phi_sig_in, prior_mu_in, prior_sig_in]) kl_t = kl_in.reshape((z_shape[0], z_shape[1])) recon = GMM(x_in, theta_mu_in, theta_sig_in, coeff_in) recon = recon.reshape((x_shape[0], x_shape[1])) recon_term = recon.mean() kl_term = kl_t.mean() nll_lower_bound = recon_term + kl_term nll_lower_bound.name = 'nll_lower_bound' recon_term.name = 'recon_term' kl_term.name = 'kl_term' kl_ratio = kl_term / T.abs_(recon_term) kl_ratio.name = 'kl_term proportion' max_x = x.max() mean_x = x.mean() min_x = x.min() max_x.name = 'max_x'