# make grid samples if Use_simple: Comb, Batch_U = grid_samples_simple(D, Grid_num) else: Comb, Batch_U = grid_samples(D, Grid_num) # Define paramaters for the model learning_rate = 0.05 batch_size = 1000 minibatch_U = 1 minibatch_Ut = 100 n_epochs = 60 data = generator([X_train, y_train], batch_size) data2 = generator([X_test, y_test], batch_size) Uniform = generator([Comb], minibatch_U) Num = 10 #prepare the placeholders dictU = make_placeholders(Num, D) X = tf.placeholder(tf.float32, [None, 2], name='X_placeholder') Y = tf.placeholder(tf.float32, [None, 1], name='Y_placeholder') #this is the place holder for the prediction _Ut = make_placeholders(minibatch_Ut, D) ################## Define your model
else: Comb, Batch_U = grid_samples(D, Grid_num) # Define paramaters for the model learning_rate = 0.01 ''' the minibatch size of data X and Y. This should be as large values as possible for numerical stability ''' minibatch_U = 1 n_epochs = 60 #hom many placeholders we use for the loss Num = 90 Uniform = generator([Comb], minibatch_U) #how many samples we use for the predictions Ut = tf.contrib.distributions.Uniform(low=0.1, high=0.8).sample(sample_shape=(1000, D)) #prepare the placeholders dictU = make_placeholders_ones(Num, D) #this is the place holder for the prediction ################## Define your model def Gauss(loc, scale, x): return tf.exp(-tf.square(x - loc) / np.sqrt(
# how many parameters are in the model we use D = 3 # Define paramaters for the model learning_rate = 0.01 ''' the minibatch size of data X and Y. This should be as large values as possible for numerical stability ''' batch_size = 1000 minibatch_U = 1 n_epochs = 60 #hom many placeholders we use for the loss Num = 50 data = generator([X_train, y_train], batch_size) data2 = generator([X_test, y_test], batch_size) #how many samples we use for the predictions #prepare the placeholders X = tf.placeholder(tf.float32, [None, 2], name='X_placeholder') Y = tf.placeholder(tf.float32, [None, 1], name='Y_placeholder') #this is the place holder for the prediction ################## Define your model def Logistic_regression_Model_prior(X, Y, params, minibatch_U=1): '''
''' batch_size = 1000 minibatch_U = 1 n_epochs = 60 #hom many placeholders we use for the loss Num = 4 n_fold = 10 # 交差検定の回数 k_fold = cross_validation.KFold(n=len(X0), n_folds=n_fold, random_state=0) for train_index, test_index in k_fold: X_train, X_test = X0[train_index, :], X0[test_index, :] X_train, X_test = preprocessing2(X_train, X_test) y_train, y_test = Y0[train_index][:, None], Y0[test_index][:, None] data = generator([X_train, y_train], batch_size) ################## Define your model ''' Bayesian neural net, one hidden layer ''' Num_of_hidden = 20 target_shape = [[Num_of_hidden, data_dim], [Num_of_hidden, 1], [1, Num_of_hidden], [1, 1]] target_shape = np.array(target_shape) a = np.cumprod(target_shape, 1)[:, -1] b = np.cumsum(np.cumprod(target_shape, 1)[:, -1]) D = np.cumsum(np.cumprod(target_shape, 1)[:, -1])[-1]
def train(model, data, batch_size, warmup_epoch, total_epoch, lr, temperature, checkpoint_path="models/checkpoints", model_summary_path="models/summary"): total_iterations = math.ceil(len(data) / batch_size) zdim = model.output.shape[1] checkpoint_path = f'models/{model.layers[1].name}_{datetime.date.today().strftime("%Y%m%d")}_checkpoints' lr_decayed_fn = tf.keras.experimental.CosineDecay(lr, total_iterations) optimizer = tf.keras.optimizers.SGD(learning_rate=lr, momentum=0.9) checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model) manager = tf.train.CheckpointManager(checkpoint, checkpoint_path, max_to_keep=10) summary_writer = tf.summary.create_file_writer(model_summary_path) images = generator(data, batch_size) # warm up head for e in range(warmup_epoch): start = time.time() epoch_loss = [] checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model) #print(f"********************************************************************") #print(f"** Warmup Epoch: {e} **") #print(f"********************************************************************") print(f"Warmup Epoch {e}: ", end='') for i in range(total_iterations): image1, image2 = next(images) # Train one step with tf.GradientTape() as tape: z1 = model(image1, training=True) z2 = model(image2, training=True) z1 = tf.math.l2_normalize(z1, axis=1) z2 = tf.math.l2_normalize(z2, axis=1) loss = nt_xent(z1, z2, batch_size, temperature, zdim) reg_loss = tf.add_n(model.losses) if model.losses else 0 loss = loss + reg_loss gradients = tape.gradient(loss, model.trainable_variables) # record loss epoch_loss.append(loss) # update optimizer optimizer.__setattr__('lr', lr_decayed_fn(i + 1)) # apply gradients optimizer.apply_gradients(zip(gradients, model.trainable_variables)) # checkpoint.step.assign_add(1) if checkpoint.step.numpy() % 10 == 0: #print(f"Iter: {i+2} Step: {checkpoint.step.numpy()} Loss: {loss.numpy():.5f} LR: {optimizer.__getattribute__('lr').numpy():9f}") add_to_summary(summary_writer, loss, optimizer.__getattribute__('lr'), image1[:1], image2[:1], checkpoint.step.numpy()) summary_writer.flush() save_path = manager.save() print("time: {:5.0f} loss {:1.5f} ".format(time.time() - start, np.mean(epoch_loss)), end='') print(f"checkpoint: {save_path}") #print("loss {:1.2f}".format(np.mean(epoch_loss))) # train all layers for l in model.layers: l.trainable = True for e in range(total_epoch): epoch_loss = [] start = time.time() checkpoint = tf.train.Checkpoint(step=tf.Variable(1), optimizer=optimizer, net=model) #print(f"********************************************************************") #print(f"** Epoch: {e} **") #print(f"********************************************************************") print(f"Epoch {e}: ", end='') for i in range(total_iterations): image1, image2 = next(images) # Train one step with tf.GradientTape() as tape: z1 = model(image1, training=True) z2 = model(image2, training=True) z1 = tf.math.l2_normalize(z1, axis=1) z2 = tf.math.l2_normalize(z2, axis=1) loss = nt_xent(z1, z2, batch_size, temperature, zdim) reg_loss = tf.add_n(model.losses) if model.losses else 0 loss = loss + reg_loss gradients = tape.gradient(loss, model.trainable_variables) # record loss epoch_loss.append(loss) # update optimizer optimizer.__setattr__('lr', lr_decayed_fn(i + 1)) # apply gradients optimizer.apply_gradients(zip(gradients, model.trainable_variables)) # checkpoint.step.assign_add(1) if checkpoint.step.numpy() % 10 == 0: #print(f"Iter: {i+2} Step: {checkpoint.step.numpy()} Loss: {loss.numpy():.5f} LR: {optimizer.__getattribute__('lr').numpy():9f}") add_to_summary(summary_writer, loss, optimizer.__getattribute__('lr'), image1[:1], image2[:1], checkpoint.step.numpy()) summary_writer.flush() print("time: {:5.0f} loss {:1.5f} ".format(time.time() - start, np.mean(epoch_loss)), end='') save_path = manager.save() print(f"Checkpoint: {save_path}") #print("loss {:1.2f}".format(loss.numpy())) return model
def Reyni_estimator(VI, dataset_name, per): name = dataset_name X0, y0, n_splits, indexes = dataload2(name) N, D = X0.shape def neural_network(X): h = tf.nn.relu(tf.matmul(X, W_0) + b_0) h = tf.nn.relu(tf.matmul(h, W_1) + b_1) h = tf.matmul(h, W_2) + b_2 return tf.reshape(h, [-1]) M = 128 # batch size during training H = 20 n_batch = int(N / M) n_epoch = 200 if VI == 'KL': gamma_list = [1] # The outlier_tuning hyper-parameter # In this code, the cross validation will not be done for simplicity. # Just separate the training dataset into training data and test data and run 10 times for each hyper parameter. elif VI == 'beta': gamma_list = [0.1, 0.2, 0.3, 0.4] var = 0.01 var2 = 1.0000 num = 10 Results = np.zeros(num) for gamma in gamma_list: # MODEL with tf.name_scope("model"): W_0 = Normal(loc=tf.zeros([D, H]), scale=tf.ones([D, H]) * var2, name="W_0") W_1 = Normal(loc=tf.zeros([H, H]), scale=tf.ones([H, H]) * var2, name="W_1") W_2 = Normal(loc=tf.zeros([H, 1]), scale=tf.ones([H, 1]) * var2, name="W_2") b_0 = Normal(loc=tf.zeros(H), scale=tf.ones(H) * var2, name="b_0") b_1 = Normal(loc=tf.zeros(H), scale=tf.ones(H) * var2, name="b_1") b_2 = Normal(loc=tf.zeros(1), scale=tf.ones(1) * var2, name="b_2") X = tf.placeholder(tf.float32, [None, D], name="X") y_ph = tf.placeholder(tf.float32, [None]) y = Normal(loc=neural_network(X), scale=1., name="y") # INFERENCE with tf.name_scope("posterior"): with tf.name_scope("qW_0"): qW_0 = Normal(loc=tf.Variable(tf.random_normal([D, H], stddev=var), name="loc"), scale=tf.nn.softplus( tf.Variable(tf.random_normal([D, H], stddev=var), name="scale"))) with tf.name_scope("qW_1"): qW_1 = Normal(loc=tf.Variable(tf.random_normal([H, H], stddev=var), name="loc"), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, H], stddev=var), name="scale"))) with tf.name_scope("qW_2"): qW_2 = Normal(loc=tf.Variable(tf.random_normal([H, 1], stddev=var), name="loc"), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H, 1], stddev=var), name="scale"))) with tf.name_scope("qb_0"): qb_0 = Normal(loc=tf.Variable(tf.random_normal([H], stddev=var), name="loc"), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H], stddev=var), name="scale"))) with tf.name_scope("qb_1"): qb_1 = Normal(loc=tf.Variable(tf.random_normal([H], stddev=var), name="loc"), scale=tf.nn.softplus( tf.Variable(tf.random_normal([H], stddev=var), name="scale"))) with tf.name_scope("qb_2"): qb_2 = Normal(loc=tf.Variable(tf.random_normal([1], stddev=var), name="loc"), scale=tf.nn.softplus( tf.Variable(tf.random_normal([1], stddev=var), name="scale"))) results = [] for data in range(num): index = indexes[data] X_train, X_test = X0[index[0], :], X0[index[1], :] y_train, y_test = y0[index[0]], y0[index[1]] X_train, y_train, X_test, y_test, mean_y_train, std_y_train = preprocessing( X_train, y_train, X_test, y_test) X_train, y_train = add_noise( X_train, y_train, noise_x=(0, 6, np.random.choice(D, D, replace=False)), noise_y=(0, 6), percent=per) data = generator([X_train, y_train], M) N = X_train.shape[0] if VI == 'KL': print("KL") inference = ed.KLqp( { W_0: qW_0, b_0: qb_0, W_1: qW_1, b_1: qb_1, W_2: qW_2, b_2: qb_2 }, data={y: y_ph}) inference.initialize(n_iter=10000, n_samples=15, scale={y: N / M}) elif VI == 'beta': print("beta") inference = KLqp_beta( { W_0: qW_0, b_0: qb_0, W_1: qW_1, b_1: qb_1, W_2: qW_2, b_2: qb_2 }, data={y: y_ph}) inference.initialize(n_iter=10000, n_samples=15, alpha=gamma, size=M, tot=N, scale={y: N / M}) tf.global_variables_initializer().run() for _ in range(inference.n_iter): X_batch, y_batch = next(data) info_dict = inference.update({X: X_batch, y_ph: y_batch}) inference.print_progress(info_dict) y_post = ed.copy(y, { W_0: qW_0, b_0: qb_0, W_1: qW_1, b_1: qb_1, W_2: qW_2, b_2: qb_2 }) print("Mean squared error on test data:") a = ed.evaluate('mean_squared_error', data={ X: X_test, y_post: y_test }) print(std_y_train * (a**0.5), a, std_y_train) print(gamma) results.append(std_y_train * (a**0.5)) results = np.array(results) Results = np.vstack((Results, results)) mu = np.mean(Results, -1) std = np.std(Results, -1) Saving = [mu, std] np.save(str('RMSE_') + str(VI) + str(name) + str(per) + '.npy', Saving)
H = tf.nn.sigmoid(tf.matmul(X, w_h) + b_h) H2 = tf.nn.sigmoid(tf.matmul(H, w_h2) + b_h2) W0 = tf.nn.sigmoid(tf.matmul(H2, w_h3) + b_h3) return W0 n_u = 10 U = tf.placeholder(tf.float32, [n_u, 1], name='U_placeholder') W = trial_solution(U) Grid_num = 100 Comb = np.arange(1, Grid_num)[:, None] np.random.shuffle(Comb) Comb = Comb / Grid_num Uniform = generator([Comb], n_u) #### Variable Transformation # Since the support of Gaussian is (-inf,inf), we transform it to the support (0,1) for the numerical boundary condition def minus_inf_inf_to_zero_one(variable): return tf.log(variable / (1 - variable)) def minus_inf_inf_to_zero_one_Log_det_J(variable): return tf.log(1 / variable * (1 - variable)) #Transform params = minus_inf_inf_to_zero_one(W) #Variable transoformation's Log_det_Jacobian
''' batch_size = 1000 minibatch_U = 1 n_epochs = 60 #hom many placeholders we use for the loss Num = 4 n_fold = 10 # 交差検定の回数 k_fold = cross_validation.KFold(n=len(X0), n_folds=n_fold, random_state=0) for train_index, test_index in k_fold: X_train, X_test = X0[train_index, :], X0[test_index, :] X_train, X_test = preprocessing2(X_train, X_test) y_train, y_test = Y0[train_index][:, None], Y0[test_index][:, None] data = generator([X_train, y_train], batch_size) X = tf.placeholder(tf.float32, [None, data_dim], name='X_placeholder') Y = tf.placeholder(tf.float32, [None, 1], name='Y_placeholder') ################## Define your model ''' Bayesian neural net, one hidden layer ''' Num_of_hidden = 20 target_shape = [[Num_of_hidden, data_dim], [Num_of_hidden, 1], [1, Num_of_hidden], [1, 1]] target_shape = np.array(target_shape) a = np.cumprod(target_shape, 1)[:, -1]