# Normalization
Y_nasa[:, 0] = Y_nasa[:, 0] - Y_nasa[:, 0].mean()
Y_nasa = Y_nasa[:, 0, np.newaxis]
mean_nasa = Y_nasa[:, 0].mean()
N = Y_nasa.shape[0]
M = 15 # initial
Q = 1
T = N
dimX = 1
max_X = 100.0
VEM_its = 1
max_iter = 100
# Likelihood Definition
likelihoods_list = [Gaussian(sigma=1.0)]
likelihood = HetLikelihood(likelihoods_list)
Y_metadata = likelihood.generate_metadata()
D = likelihood.num_output_functions(Y_metadata)
true_W_list = [np.array(([[1.0]]))]
# Streaming data generator:
def data_streaming(Y_output, X_input, T_sections):
streaming_Y = []
streaming_X = []
N = Y_output.shape[0]
slice = np.floor(N / T_sections)
for t in range(T_sections):
streaming_X.append(X_input[np.r_[int(t * slice):int((t + 1) *
slice)], :])
###########################
# #
# PARALLEL INFERENCE #
# #
###########################
N_k_test = 400
x_test = torch.linspace(min_x - 0.5, max_x + 0.5, N_k_test)[:, None]
models = []
for k, x_k in enumerate(x_tasks):
print('- -')
print('----- TASK k=' + str(k + 1) + ' ------')
print('- -')
kernel_k = RBF()
likelihood_k = Gaussian(fit_noise=False)
model_k = SVGP(kernel_k, likelihood_k, M_k)
z_k_min = min_x + (k * segment_x)
z_k_max = min_x + ((k + 1) * segment_x)
#model_k.z = torch.nn.Parameter((z_k_max - z_k_min)*torch.rand(M_k, 1) + z_k_min, requires_grad=True)
model_k.z = torch.nn.Parameter(torch.linspace(z_k_min, z_k_max, M_k)[:,
None],
requires_grad=True)
vem_algorithm = AlgorithmVEM(model_k, x_k, y_tasks[k], iters=15)
vem_algorithm.ve_its = 20
vem_algorithm.vm_its = 10
vem_algorithm.lr_m = 1e-6
vem_algorithm.lr_L = 1e-10
vem_algorithm.lr_hyp = 1e-10
def test_model(N,Dc,Dd,Db,L,K,X_c=None,X_d=None,X_b=None):
batch_size = int(N/4)
epsilon = 1e0
# ----------- Model ------------
gaussian = Gaussian(Dc, L, K)
categorical = Categorical(Dd, L, K)
bernoulli = Bernoulli(Db, L, K)
likelihoods = [gaussian,bernoulli,categorical]
model = Mixture_Model(N, L, likelihoods)
optim = torch.optim.Adagrad(model.parameters(), lr=0.01)
autograd.set_detect_anomaly(True)
# optim = torch.optim.SGD(model.parameters(),lr=0.001, momentum= 0.9)
data_set = torch.utils.data.TensorDataset(torch.Tensor(X_c), torch.Tensor(X_d),torch.Tensor(X_b))
#data_set = torch.utils.data.TensorDataset(torch.Tensor(X_c),torch.Tensor(X_b))
data_loader = torch.utils.data.DataLoader(data_set, batch_size=batch_size, shuffle=False) # shuffle a true?
#data_loader= torch.utils.data.DataLoader(X_c, batch_size = batch_size, shuffle=False) #shuffle a true?
num_epochs = 100
ll_list = []
loss_list = []
KL_z_list = []
KL_s_list = []
rik_epochs = []
term_1_list = []
term_2_list = []
term_3_list = []
past_loss = 0
for epoch in range(num_epochs):
loss_epoch = 0
ll_epoch = 0
KL_z_epoch = 0
KL_s_epoch = 0
term_1_epoch = 0
term_2_epoch = 0
term_3_epoch = 0
# for x_batch_real, x_batch_discrete in data_loader:
for index, x_batch in enumerate(data_loader):
x_batch_real = x_batch[0]
x_batch_disc = x_batch[1]
x_batch_bin = x_batch[2]
# ----- Variational E ----- fix θ
optim.zero_grad()
util.fix_model_params(likelihoods, set=False)
util.fix_variational_params(model, set=True)
loss, LL, KL_z, KL_s, rik, term_1, term_2,term_3 = model(index, X_c=x_batch_real.numpy(), X_d=x_batch_disc.numpy(), X_b=x_batch_bin.numpy())
loss.backward()
optim.step()
# ----- Variational M ----- fix φ
optim.zero_grad()
util.fix_model_params(likelihoods, set=True)
util.fix_variational_params(model, set=False)
loss, LL, KL_z, KL_s, rik, term_1, term_2,term_3 = model(index, X_c=x_batch_real.numpy(), X_d=x_batch_disc.numpy(), X_b=x_batch_bin.numpy())
loss.backward()
optim.step()
ll_epoch += LL
KL_s_epoch += KL_s
KL_z_epoch += KL_z
loss_epoch += loss
term_1_epoch += term_1
term_2_epoch += term_2
term_3_epoch += term_3
#print(f"Epoch = {epoch}, Loglik ={ll_epoch}, -ELBO ={loss_epoch}")
rik_epochs.append(rik)
KL_z_list.append(KL_z_epoch)
KL_s_list.append(KL_s_epoch)
loss_list.append(loss_epoch)
term_1_list.append(term_1_epoch)
term_2_list.append(term_2_epoch)
term_3_list.append(term_3_epoch)
ll_list.append(ll_epoch)
z_mean = model.q_z_mean
W_c = model.gaussian.W_c
var_c =model.gaussian.var_c
W_b = model.bernoulli.W_d
W_d = model.categorical.W_d
#W_d = None
mu_d = model.categorical.mu_d
#mu_d = None
mu_b = model.bernoulli.mu_d
param = torch.nn.functional.softmax(model.q_s_param, dim=1).detach().numpy()
#print(param)
profiles = np.argmax(param, axis=1) + 1
'''
plt.figure()
plt.plot(np.arange(num_epochs), KL_z_list)
plt.title(f'Convergence of KL_z for K={K}')
plt.xlabel('Epochs')
plt.ylabel('Kullback-Leibler divergence')
plt.savefig('KL_z_'+str(K)+'.png')
plt.figure()
plt.plot(np.arange(num_epochs), KL_s_list)
plt.title(f'Convergence of KL_s for K={K}')
plt.xlabel('Epochs')
plt.ylabel('Kullback-Leibler divergence')
plt.savefig('KL_s_'+str(K)+'.png')
'''
plt.figure()
plt.plot(np.arange(num_epochs), term_1_list)
plt.title(f'Convergence of ELBO terms for K={K}')
plt.legend([ 'Gaussian Term '])
plt.xlabel('Epochs')
plt.ylabel('Likelihood')
plt.savefig('GaussianTerm_'+str(K)+'.png')
plt.figure()
plt.plot(np.arange(num_epochs), term_2_list)
plt.title(f'Convergence of ELBO terms for K={K}')
plt.legend(['Bernoulli term'])
plt.xlabel('Epochs')
plt.ylabel('Likelihood')
plt.savefig('BernoulliTerm_'+str(K)+'.png')
plt.figure()
plt.plot(np.arange(num_epochs), term_3_list)
plt.title(f'Convergence of ELBO terms for K={K}')
plt.legend(['Categorical term'])
plt.xlabel('Epochs')
plt.ylabel('Likelihood')
plt.savefig('CategoricalTerm_'+str(K)+'.png')
plt.figure()
plt.plot(np.arange(num_epochs), ll_list)
plt.plot(np.arange(num_epochs), loss_list)
plt.title(f'Performance in epochs for K={K}')
plt.legend(['Likelihood evolution', 'Loss evolution'])
plt.xlabel('Epochs')
plt.ylabel('Likelihood')
plt.savefig('Convergence_'+str(K)+'.png')
#plt.show()
return ll_list[-1],z_mean,W_c,W_b,mu_b,mu_d,W_d,var_c,profiles
from continualgp.util import draw_mini_slices
from hetmogp.svmogp import SVMOGP
from hetmogp.util import vem_algorithm as VEM
warnings.filterwarnings("ignore")
os.environ['PATH'] = os.environ['PATH'] + ':/usr/texbin'
N = 2000 # number of samples
M = 5 # number of inducing points
Q = 2 # number of latent functions
T = 5 # number of streaming batches
max_X = 2.0 # max of X range
VEM_its = 2
# Heterogeneous Likelihood Definition
likelihoods_list = [Gaussian(sigma=1.), Gaussian(sigma=2.0)]
likelihood = HetLikelihood(likelihoods_list)
Y_metadata = likelihood.generate_metadata()
D = likelihood.num_output_functions(Y_metadata)
X1 = np.sort(max_X * np.random.rand(N))[:, None]
X2 = np.sort(max_X * np.random.rand(N))[:, None]
X = [X1, X2]
# True U functions
def true_u_functions(X_list):
u_functions = []
for X in X_list:
u_task = np.empty((X.shape[0], 2))
u_task[:, 0, None] = 4.5 * np.cos(2 * np.pi * X + 1.5 * np.pi) - \
from hetmogp.svmogp import SVMOGP
from hetmogp.util import vem_algorithm as VEM
warnings.filterwarnings("ignore")
os.environ['PATH'] = os.environ['PATH'] + ':/usr/texbin'
N = 2000 # number of samples
M = 3 # number of inducing points
Q = 1 # number of latent functions
T = 10 # number of streaming batches
max_X = 2.0 # max of X range
VEM_its = 2
max_iter = 100
# Likelihood Definition
likelihoods_list = [Gaussian(sigma=1.5)]
likelihood = HetLikelihood(likelihoods_list)
Y_metadata = likelihood.generate_metadata()
D = likelihood.num_output_functions(Y_metadata)
X = np.sort(max_X * np.random.rand(N))[:, None]
# True F function
def true_f(X_input):
f = np.empty((X_input.shape[0], 1))
f[:,0,None] = 4.5 * np.cos(2 * np.pi * X_input + 1.5*np.pi) - \
3 * np.sin(4.3 * np.pi * X_input + 0.3 * np.pi) + \
5 * np.cos(7 * np.pi * X_input + 2.4 * np.pi)
return [f]
def load_toy3(N=1000, input_dim=1):
if input_dim == 2:
Nsqrt = int(N**(1.0 / input_dim))
print('input_dim:', input_dim)
# Heterogeneous Likelihood Definition
likelihoods_list = [
HetGaussian(),
Beta(),
Bernoulli(),
Gamma(),
Exponential(),
Gaussian(sigma=0.1),
Beta(),
Bernoulli(),
Gamma(),
Exponential()
] # Real + Binary
likelihood = HetLikelihood(likelihoods_list)
Y_metadata = likelihood.generate_metadata()
D = likelihoods_list.__len__()
Q = 3
"""""" """""" """""" """""" """"""
Dim = input_dim
if input_dim == 2:
xy = np.linspace(0.0, 1.0, Nsqrt)
xx = np.linspace(0.0, 1.0, Nsqrt)
XX, XY = np.meshgrid(xx, xy)
XX = XX.reshape(Nsqrt**2, 1)
XY = XY.reshape(Nsqrt**2, 1)
Xtoy = np.hstack((XX, XY))
else:
minis = 0 * np.ones(Dim)
maxis = 1 * np.ones(Dim)
Xtoy = np.linspace(minis[0], maxis[0], N).reshape(1, -1)
for i in range(Dim - 1):
Xaux = np.linspace(minis[i + 1], maxis[i + 1], N)
Xtoy = np.concatenate(
(Xtoy, Xaux[np.random.permutation(N)].reshape(1, -1)), axis=0)
# Z = np.concatenate((Z, Zaux.reshape(1, -1)), axis=0)
Xtoy = 1.0 * Xtoy.T
def latent_functions_prior(Q,
lenghtscale=None,
variance=None,
input_dim=None):
if lenghtscale is None:
lenghtscale = np.array(
[0.5, 0.05,
0.1]) #This is the one used for previous experiments
else:
lenghtscale = lenghtscale
if variance is None:
variance = 1 * np.ones(Q)
else:
variance = variance
kern_list = []
for q in range(Q):
kern_q = GPy.kern.RBF(input_dim=input_dim,
lengthscale=lenghtscale[q],
variance=variance[q],
name='rbf')
kern_q.name = 'kern_q' + str(q)
kern_list.append(kern_q)
return kern_list
kern_list = latent_functions_prior(Q,
lenghtscale=None,
variance=None,
input_dim=Dim)
# True U and F functions
def experiment_true_u_functions(kern_list, X):
Q = kern_list.__len__()
u_latent = np.zeros((X.shape[0], Q))
np.random.seed(104)
for q in range(Q):
u_latent[:, q] = np.random.multivariate_normal(
np.zeros(X.shape[0]), kern_list[q].K(X))
return u_latent
def experiment_true_f_functions(true_u, X_list, J):
Q = true_u.shape[1]
W = W_lincombination(Q, J)
#print(W)
#for j in range(J):
f_j = np.zeros((X_list.shape[0], J))
for q in range(Q):
f_j += (W[q] * true_u[:, q]).T
#true_f.append(f_d)
return f_j
# True Combinations
def W_lincombination(Q, J):
W_list = []
# q=1
for q in range(Q):
if q == 0:
W_list.append(
np.array([
-0.1, -0.1, 1.1, 2.1, -1.1, -0.5, -0.6, 0.1, -1.1, 0.8,
1.5, -0.2, 0.05, 0.06, 0.3
])[:, None])
elif q == 1:
W_list.append(
np.array([
1.4, -0.5, 0.3, 0.7, 1.5, -0.3, 0.4, -0.2, 0.4, 0.3,
-0.7, -2.1, -0.03, 0.04, -0.5
])[:, None])
else:
W_list.append(
np.array([
0.1, -0.8, 1.3, 1.5, 0.5, -0.02, 0.01, 0.5, 0.5, 1.0,
0.8, 3.0, 0.1, -0.5, 0.4
])[:, None])
return W_list
"""""" """""" """""" """""" """""" ""
# True functions values for inputs X
f_index = Y_metadata['function_index'].flatten()
J = f_index.__len__()
trueU = experiment_true_u_functions(kern_list, Xtoy)
trueF = experiment_true_f_functions(trueU, Xtoy, J)
d_index = Y_metadata['d_index'].flatten()
F_true = []
for t in range(D):
_, num_f_task, _ = likelihoods_list[t].get_metadata()
f = np.empty((Xtoy.shape[0], num_f_task))
for j in range(J):
if f_index[j] == t:
f[:, d_index[j], None] = trueF[:, j][:, None]
F_true.append(f)
# Generating training data Y (sampling from heterogeneous likelihood)
Ytrain = likelihood.samples(F=F_true, Y_metadata=Y_metadata)
Yreg1 = (Ytrain[0] - Ytrain[0].mean(0)) / (Ytrain[0].std(0))
Yreg2 = (Ytrain[5] - Ytrain[5].mean(0)) / (Ytrain[5].std(0))
Ytrain = [
Yreg1,
np.clip(Ytrain[1], 1.0e-9, 0.99999), Ytrain[2], Ytrain[3], Ytrain[4],
Yreg2,
np.clip(Ytrain[6], 1.0e-9, 0.99999), Ytrain[7], Ytrain[8], Ytrain[9]
]
Xtrain = []
for d in range(likelihoods_list.__len__()):
Xtrain.append(Xtoy)
return Xtrain, Ytrain
def load_toy1_conv(N=1000, input_dim=1):
if input_dim == 2:
Nsqrt = int(N**(1.0 / input_dim))
print('input_dim:', input_dim)
# Q = 5 # number of latent functions
# Heterogeneous Likelihood Definition
# likelihoods_list = [Gaussian(sigma=1.0), Bernoulli()] # Real + Binary
likelihoods_list = [Gaussian(sigma=0.1),
Gaussian(sigma=0.1)] # Real + Binary
# likelihoods_list = [Gaussian(sigma=1.0)]
likelihood = HetLikelihood(likelihoods_list)
Y_metadata = likelihood.generate_metadata()
D = likelihoods_list.__len__()
J = likelihood.num_output_functions(Y_metadata)
Q = 1
"""""" """""" """""" """""" """"""
rescale = 10
Dim = input_dim
if input_dim == 2:
xy = rescale * np.linspace(-1.0, 1.0, Nsqrt)
xx = rescale * np.linspace(-1.0, 1.0, Nsqrt)
XX, XY = np.meshgrid(xx, xy)
XX = XX.reshape(Nsqrt**2, 1)
XY = XY.reshape(Nsqrt**2, 1)
Xtoy = np.hstack((XX, XY))
else:
minis = -rescale * np.ones(Dim)
maxis = rescale * np.ones(Dim)
Xtoy = np.linspace(minis[0], maxis[0], N).reshape(1, -1)
for i in range(Dim - 1):
Xaux = np.linspace(minis[i + 1], maxis[i + 1], N)
Xtoy = np.concatenate(
(Xtoy, Xaux[np.random.permutation(N)].reshape(1, -1)), axis=0)
# Z = np.concatenate((Z, Zaux.reshape(1, -1)), axis=0)
Xtoy = 1.0 * Xtoy.T
def latent_functions_prior(Q,
lengthscale=None,
input_dim=None,
name='kern_q'):
#lenghtscale = rescale * np.array([0.5]) # This is the one used for previous experiments
#lenghtscale = rescale * lengthscale
lenghtscale = lengthscale
variance = 1 * np.ones(Q)
kern_list = []
for q in range(Q):
# print("length:",lenghtscale[q])
# print("var:", variance[q])
kern_q = GPy.kern.RBF(input_dim=input_dim,
lengthscale=lenghtscale[q],
variance=variance[q],
name='rbf')
kern_q.name = name + str(q)
kern_list.append(kern_q)
return kern_list
kern_list_uq = latent_functions_prior(Q,
lengthscale=np.array([0.1]),
input_dim=Dim,
name='kern_q')
kern_list_Gdj = latent_functions_prior(J,
lengthscale=np.array([0.3, 0.7]),
input_dim=Dim,
name='kern_G')
kern_aux = GPy.kern.RBF(input_dim=Dim,
lengthscale=1.0,
variance=1.0,
name='rbf_aux',
ARD=False) + GPy.kern.White(input_dim=Dim)
kern_aux.white.variance = 1e-6
# True U and F functions
def experiment_true_u_functions(kern_list, kern_list_Gdj, X):
Q = kern_list.__len__()
# for d,X in enumerate(X_list):
u_latent = np.zeros((X.shape[0], Q))
np.random.seed(104)
for q in range(Q):
u_latent[:, q] = np.random.multivariate_normal(
np.zeros(X.shape[0]), kern_list[q].K(X))
return u_latent
def experiment_true_f_functions(kern_u, kern_G, kern_aux, X, Q, J):
W = W_lincombination(Q, J)
# print(W)
# for j in range(J):
f_j = np.zeros((X.shape[0], J))
for j in range(J):
for q in range(Q):
util.update_conv_Kff(kern_u[q], kern_G[j], kern_aux)
#f_j += (W[q] * true_u[:, q]).T
f_j[:, j] += W[q][j] * np.random.multivariate_normal(
np.zeros(X.shape[0]), kern_aux.K(X))
# true_f.append(f_d)
return f_j
# True Combinations
def W_lincombination(Q, J):
W_list = []
# q=1
for q in range(Q):
# W_list.append(np.array(([[-0.5], [0.1]])))
if q == 0:
# W_list.append(0.3*np.random.randn(J, 1))
W_list.append(np.array([-0.5, 2.1])[:, None])
elif q == 1:
# W_list.append(2.0 * np.random.randn(J, 1))
W_list.append(np.array([
1.4,
0.3,
])[:, None])
else:
# W_list.append(10.0 * np.random.randn(J, 1)+0.1)
W_list.append(np.array([
0.1,
-0.8,
])[:, None])
return W_list
"""""" """""" """""" """""" """""" ""
# True functions values for inputs X
f_index = Y_metadata['function_index'].flatten()
J = f_index.__len__()
trueF = experiment_true_f_functions(kern_list_uq, kern_list_Gdj, kern_aux,
Xtoy, Q, J)
# if input_dim==2:
# #from mpl_toolkits.mplot3d import Axes3D # noqa: F401 unused import
#
# from matplotlib import cm
# from matplotlib.ticker import LinearLocator, FormatStrFormatter
# fig = plt.figure()
# ax = fig.gca(projection='3d')
#
# # Make data.
# # X = np.arange(-5, 5, 0.25)
# # Y = np.arange(-5, 5, 0.25)
# # X, Y = np.meshgrid(X, Y)
# # R = np.sqrt(X ** 2 + Y ** 2)
# # Z = np.sin(R)
#
# # Plot the surface.
# surf = ax.plot_surface(Xtoy[:,0].reshape(Nsqrt,Nsqrt), Xtoy[:,1].reshape(Nsqrt,Nsqrt), trueF[:,2].reshape(Nsqrt,Nsqrt), cmap=cm.coolwarm,linewidth=0, antialiased=False)
#
# # Customize the z axis.
# #ax.set_zlim(-1.01, 1.01)
# ax.zaxis.set_major_locator(LinearLocator(10))
# ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
#
# # Add a color bar which maps values to colors.
# fig.colorbar(surf, shrink=0.5, aspect=5)
#
# plt.show()
#
# else:
# plt.figure(15)
# plt.plot(trueF[:,1])
# plt.figure(16)
# plt.plot(trueU)
d_index = Y_metadata['d_index'].flatten()
F_true = []
# for i,f_latent in enumerate(trueF):
# if
for t in range(D):
_, num_f_task, _ = likelihoods_list[t].get_metadata()
f = np.empty((Xtoy.shape[0], num_f_task))
for j in range(J):
if f_index[j] == t:
f[:, d_index[j], None] = trueF[:, j][:, None]
F_true.append(f)
# Generating training data Y (sampling from heterogeneous likelihood)
Ytrain = likelihood.samples(F=F_true, Y_metadata=Y_metadata)
#Yreg0 = (Ytrain[0] - Ytrain[0].mean(0)) / (Ytrain[0].std(0))
#Yreg1 = (Ytrain[1] - Ytrain[1].mean(0)) / (Ytrain[1].std(0))
#Ytrain = [Yreg0, Yreg1]
Xtrain = []
for d in range(likelihoods_list.__len__()):
Xtrain.append(Xtoy)
return Xtrain, Ytrain
HetGaussian(),
Beta(),
Bernoulli(),
Gamma(),
Exponential()
]
elif dataset == 'toy3':
Q = 3
prior_lamb = 1
likelihoods_list = [
HetGaussian(),
Beta(),
Bernoulli(),
Gamma(),
Exponential(),
Gaussian(sigma=0.1),
Beta(),
Bernoulli(),
Gamma(),
Exponential()
]
elif dataset == 'toy4':
Q = 3
prior_lamb = 1
likelihoods_list = [HetGaussian(), Beta()]
elif dataset == 'toy5':
Q = 3
prior_lamb = 1
likelihoods_list = [HetGaussian(), Beta(), Gamma()]
if 'toy' in dataset: