def _each_loss(X, Y, Num, Model_prior, D, F, U): #U=[tf.contrib.distributions.Uniform().sample(sample_shape=(1,1)) for j in range (D)] _W = [f(u) for f, u in zip(F, U)] #variable transformation W = [minus_inf_inf_to_zero_one(__w) for __w in _W] raw_params = _W #Variable transoformation's Log_det_Jacobian minus_Log_det_J = 0 for i in raw_params: minus_Log_det_J += tf.log( tf.abs(tf.reduce_sum(minus_inf_inf_to_zero_one_Log_det_J(i), 1))) #likelihood and prior calculation #if you want to consider other than logistic regression, change the model and prior Log_likelihood, Prior, _ = Model_prior(X, Y, W) Posteri_numerator = Log_likelihood[:, None] + Prior + minus_Log_det_J[:, None] dp_du = [tf.gradients(Posteri_numerator, u)[0] for u in U] d2w_du2 = [ tf.gradients(tf.log(tf.abs(tf.gradients(_p, __u)[0])), __u)[0] for _p, __u in zip(raw_params, U) ] _G = [(i + j)**2 for i, j in zip(d2w_du2, dp_du)] L_0 = tf.reduce_sum(_G) return L_0
def _each_loss(X, Y, Num, Model_prior, D, F, U): _W = [f(u) for f, u in zip(F, U)] #variable transformation W = [minus_inf_inf_to_zero_one(__w) for __w in _W] raw_params = _W #Variable transoformation's Log_det_Jacobian minus_Log_det_J = 0 for i in raw_params: minus_Log_det_J += -tf.reduce_sum( minus_inf_inf_to_zero_one_Log_det_J(i), 1) #likelihood and prior calculation #if you want to consider other than logistic regression, change the model and prior params = parameter_reshaper(W, b, data_dim, Num_of_hidden) Minus_Log_likelihood, Minus_Prior = Model_prior(X, Y, params) Posteri_numerator = Minus_Log_likelihood[:, None] + Minus_Prior + minus_Log_det_J[:, None] dp_du = tf.gradients(Posteri_numerator, U) d2w_du2 = [ tf.gradients(tf.log(tf.abs(tf.gradients(_p, __u)[0])), __u)[0] for _p, __u in zip(raw_params, U) ] _G = [(i - j)**2 for i, j in zip(d2w_du2, dp_du)] L_0 = tf.reduce_sum(_G) return L_0
def construct_loss(X, Y, dic, Num, Model_prior, D): ''' This is the function calculating the loss function. X,Y : data, they are place holders dic : this is the ordered dictionally which contains placeholders of U Num : how many placeholders are included in dic Model_prior : what kind of model we use D : how many parameters are in the model ''' F = make_trials(D) loss = 0 for i in range(0, Num * D, D): U = [dic["U_" + str(i + j)] for j in range(D)] _W = [f(u) for f, u in zip(F, U)] #variable transformation W = [minus_inf_inf_to_zero_one(__w) for __w in _W] raw_params = _W #Variable transoformation's Log_det_Jacobian minus_Log_det_J = 0 for i in raw_params: minus_Log_det_J += -tf.reduce_sum( minus_inf_inf_to_zero_one_Log_det_J(i), 1) #likelihood and prior calculation #if you want to consider other than logistic regression, change the model and prior params = parameter_reshaper(W, b, data_dim, Num_of_hidden) Minus_Log_likelihood, Minus_Prior = Model_prior(X, Y, params) Posteri_numerator = Minus_Log_likelihood[:, None] + Minus_Prior + minus_Log_det_J[:, None] dp_du = [tf.gradients(Posteri_numerator, u)[0] for u in U] d2w_du2 = [ tf.gradients(tf.log(tf.abs(tf.gradients(_p, __u)[0])), __u)[0] for _p, __u in zip(raw_params, U) ] _G = [(i - j)**2 for i, j in zip(d2w_du2, dp_du)] L_0 = tf.reduce_sum(_G) loss += L_0 return loss, F
def construct_loss(X, Y, F, dic, Num, Model_prior, D): ''' This is the function calculating the loss function. X,Y : data, they are place holders dic : this is the ordered dictionally which contains placeholders of U Num : how many placeholders are included in dic Model_prior : what kind of model we use D : how many parameters are in the model ''' loss = 0 for i in range(0, Num): U = dic["U_" + str(i)] _W, sigma_list = F(U) _W = tf.nn.sigmoid(_W) #variable transformation W = minus_inf_inf_to_zero_one(_W) #Variable transoformation's Log_det_Jacobian Log_det_J = tf.log( tf.abs(tf.reduce_sum(minus_inf_inf_to_zero_one_Log_det_J(_W), 1))) #likelihood and prior calculation #if you want to consider other than logistic regression, change the model and prior Log_likelihood, Prior = Model_prior(X, Y, W) Posteri_numerator = Log_likelihood[:, None] + Prior + Log_det_J[:, None] dp_du = tf.gradients(Posteri_numerator, U)[0] log_det_J = tf.reduce_sum(tf.log(sigma_list)) d2w_du2 = tf.gradients(log_det_J, U)[0] L_0 = tf.reduce_sum((d2w_du2 + dp_du)**2) loss += L_0 return loss
def construct_loss(dic,Num,Model_prior,D): ''' This is the function calculating the loss function. X,Y : data, they are place holders dic : this is the ordered dictionally which contains placeholders of U Num : how many placeholders are included in dic Model_prior : what kind of model we use D : how many parameters are in the model ''' F=make_trials(D) loss = 0 for i in range(0,Num*D,D): U=[dic["U_"+ str(i+j)] for j in range (D)] _W=[f(u) for f,u in zip(F,U)] #variable transformation W=[minus_inf_inf_to_zero_one(__w) for __w in _W] raw_params=_W #Variable transoformation's Log_det_Jacobian minus_Log_det_J=0 for i in raw_params: minus_Log_det_J+=tf.log(tf.abs(tf.reduce_sum(minus_inf_inf_to_zero_one_Log_det_J(i),1))) #likelihood and prior calculation #if you want to consider other than logistic regression, change the model and prior Log_joint=-0.5*(W[0]**2) Posteri_numerator=Log_joint+minus_Log_det_J[:,None]+np.log(1/np.sqrt((2*np.pi*np.square(1.)))) dw_du=[tf.log(tf.abs(tf.gradients(_p,__u)[0])) for _p,__u in zip(raw_params,U)] #_G=[(i+j-1)**2 for i,j in zip(dw_du,Posteri_numerator)] _G=(dw_du[0]+Posteri_numerator)**2 L_0=tf.reduce_sum(_G) loss +=L_0 return loss,F
loss += L_0 return loss F = MADE_NN(3, 8) #F=make_trials(3,MF=False) loss = construct_loss(X, Y, F, dictU, Num, Logistic_regression_Model_prior, D) optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss) Ut = tf.contrib.distributions.Uniform().sample(sample_shape=(300, D)) _Wt, _ = F(Ut) Wt = minus_inf_inf_to_zero_one(_Wt) model_output_t = Logistic_regression_Model_output(X, Wt) A = tf.reduce_mean(tf.sigmoid(model_output_t), 1) prediction = tf.round(A) predictions_correct = tf.cast(tf.equal(prediction[:, None], Y), tf.float32) accuracy = tf.reduce_mean(predictions_correct) ####sample _Us = tf.contrib.distributions.Uniform().sample(sample_shape=(10000, D)) _Ws, _ = F(_Us) Ws = minus_inf_inf_to_zero_one(_Ws) if True: sess = tf.InteractiveSession() #writer = tf.summary.FileWriter('./graphs/logistic_reg', sess.graph)
loss = _loop(each_loss, 50, 3) + _w_t**2 + (_w_t2 - 1)**2 ''' loss=0 for _ in range(20): _U=tf.contrib.distributions.Uniform().sample(sample_shape=(1,D)) U=tf.split(axis = 1, num_or_size_splits = D, value = _U) loss+=each_loss(U) ''' # Actual Prediction for test data _Ut = tf.contrib.distributions.Uniform().sample(sample_shape=(300, D)) Ut = tf.split(axis=1, num_or_size_splits=D, value=_Ut) _w_t = [f(u) for f, u in zip(F, Ut)] Wt = [minus_inf_inf_to_zero_one(__w) for __w in _w_t] model_output_t = Logistic_regression_Model_output(X, Wt) A = tf.reduce_mean(tf.sigmoid(model_output_t), 1) prediction = tf.round(A) predictions_correct = tf.cast(tf.equal(prediction[:, None], Y), tf.float32) accuracy = tf.reduce_mean(predictions_correct) optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss) ####sample _Us = tf.contrib.distributions.Uniform().sample(sample_shape=(10000, D)) Us = tf.split(axis=1, num_or_size_splits=D, value=_Us) _w_s = [f(u) for f, u in zip(F, Us)]
U_B = tf.placeholder(tf.float32, [None, 1], name='U_B_placeholder') U_B2 = tf.placeholder(tf.float32, [None, 1], name='U_B2_placeholder') BC0_data = np.array([[0.]]) BC1_data = np.array([[1.]]) _loss, F = construct_loss(dictU, Num, MoG, D) _w1_B0 = F[0](U_B) _w1_B1 = F[0](U_B2) loss = _loss + _w1_B0**2 + (_w1_B1 - 1.)**2 #+tf.losses.get_regularization_loss() _Z = F[0](Ut) Z = minus_inf_inf_to_zero_one(_Z) optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss) with tf.Session() as sess: #writer = tf.summary.FileWriter('./graphs/logistic_reg', sess.graph) start_time = time.time() sess.run(tf.global_variables_initializer()) n_batches = 50 #int(N_train/batch_size) U_batches = 50 #int(Batch_U/minibatch_U) for i in range(1): # train the model n_epochs times print(i) for _ in range(500): total_loss = 0 for _ in range(U_batches):