import tensorflow as tf
train_size = 500
T1, T2, T3, T4 = 784, 40, 40, 1
DATAPATH = os.path.join(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data')
if not os.path.exists(DATAPATH):
DATAPATH = os.path.join(
os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),
'data')
x_train_genuine = np.load(os.path.join(DATAPATH,
'train_x_20000.npy'))[:train_size]
x_test = np.load(os.path.join(DATAPATH, 'test_x_1000.npy'))[:100]
xs = np.concatenate((x_train_genuine, x_test), axis=0)
K = kernel_matrix(xs, number_layers=2, sigmaw=np.sqrt(2), sigmab=1, n_gpus=0)
volume = np.load('volume_ys_list_1.npy')
volume_list = []
for i in range(len(volume)):
tf.reset_default_graph()
print('Here we are going with %04d epoch' % i)
ys = volume[i]
logPU = GP_prob(K, xs, ys)
volume_list.append(logPU)
np.save('volume_list.npy', volume_list)
# X.shape
from nngp_kernel.fc_kernel import kernel_matrix
Kfull = kernel_matrix(Xfull, number_layers=2,sigmaw=1.0,sigmab=1.0)
K = kernel_matrix(X, number_layers=2,sigmaw=1.0,sigmab=1.0)
from GP_prob.GP_prob_gpy import GP_prob
kern_multiplier = 100
#%%
# test_ys
#
# test_ys = np.array([float(x) for x in test_funs[0]])
GP_prob_train_set = GP_prob(kern_multiplier*K,X,Y, method="EP")
logPUs_GP = []
for test_fun in test_funs:
ysfull = ys2 + [[float(x)] for x in test_fun]
Yfull = np.array(ysfull)
logPU = GP_prob(kern_multiplier*Kfull,Xfull,Yfull, method="EP") - GP_prob_train_set
logPUs_GP.append(logPU)
# logPU
import pickle
pickle.dump(logPUs_GP,open("logPUs_GP_7_2x40_1_fc_32___8_sgd_ce_kernmult100.p","wb"))
logPUs_GP = pickle.load(open("logPUs_GP_7_2x40_1_fc_32___8_sgd_ce_kernmult100.p","rb"))
Kfull = load_kernel_by_filename("kernels/"+filename)
m = 1000
# K = Kfull[0:m,0:m]
# Kfull.max()
K = Kfull/Kfull.max()
# Kfull = K
K.shape
# K
# Y = Y*2-1
Y.shape
Kfull.shape
# logdet = np.sum(np.log(np.linalg.eigh(Kfull)[0]))
# thing = -0.5*(-np.matmul(Y.T,np.matmul(np.linalg.inv(Kfull),Y)) - m*np.log(np.pi) - logdet)
from GP_prob.GP_prob_gpy import GP_prob
logPU = GP_prob(Kfull,X,Y, method="Laplace")
thing = logPU
things.append(thing)
things
#%%
filename = net+"_True_4_None_0000_max_gaussian_model"
json_string_filename = filename
arch_json_string = open("archs/"+filename, "r") .read()
from tensorflow.keras.models import model_from_json
model = model_from_json(arch_json_string)
model.compile("sgd",loss=lambda target, pred: pred )
import tensorflow.keras.backend as K
def get_weight_grad(model, inputs, outputs):
""" Gets gradient of model for given inputs and outputs for all weights"""
print(PQratio)
tot += PQratio
tots = comm.gather(tot, root=0)
if rank == 0:
tot = sum(tots)
PU = tot / num_post_samples
logPU = np.log(PU) - shift
# logPU = np.log(PU)
print(logPU)
#%%
from GP_prob.GP_prob_gpy import GP_prob
logPU = GP_prob(K, X, Y)
print(logPU)
####
#%%
import cupy as np
# import numpy as np
Kcu = np.array(K)
Ycu = np.array(Y)
PU = 0
for i in range(100):
N = 1e6
# exact_samples = np.random.multivariate_normal(np.zeros(m),K,int(N))>0
device, # Use fc-poped model
sigmaw=np.sqrt(2),
sigmab=1.0,
n_gpus=1,
empirical_kernel_batch_size=256,
truncated_init_dist=False,
store_partial_kernel=False,
partial_kernel_n_proc=1,
partial_kernel_index=0)
K = np.array(K.cpu())
np.save(os.path.join(args.path, 'empirical_K.npy'), K)
(xs, _) = get_xs_ys_from_dataset(data_train_plus_test, 256,
config.num_workers)
ys = (model_predict(net, data_train_plus_test, 256,
config.num_workers, device) > 0)
logPU = GP_prob(K, np.array(xs), np.array(ys.cpu()))
# Note: you have to use np.array(xs) instead of xs!!!
# This stupid hidden-bug has wasted my whole night!!!
# Also a funny bug: if K is a tensor on GPU and xs,ys are np.array on cpu,
# there would be problem as well!
log_10PU = logPU * np.log10(np.e)
if config.record == True:
generalization = test_loss_acc_list[-1][1] # last test accuracy
record = [generalization, np.log10(sharpness_one_off), log_10PU]
np.save(os.path.join(args.path, 'record_%d.npy' % args.sample),
record)
else: # Calculate empirical kernel only
if os.path.isfile(os.path.join(args.path, 'empirical_K.npy')):
print('Empirical Kernel Already Exist!!!:')
else: