def __init__(self,
X,
Y,
likelihood,
layers,
minibatch_size=None,
num_samples=1,
num_data=None,
**kwargs):
"""
Base class for the fully coupled DGP providing all basic functionalities.
"""
Model.__init__(self, **kwargs)
self.num_samples = num_samples
self.num_data = num_data or X.shape[0]
if minibatch_size:
self.X = Minibatch(X, minibatch_size, seed=0)
self.Y = Minibatch(Y, minibatch_size, seed=0)
else:
self.X = DataHolder(X)
self.Y = DataHolder(Y)
self.likelihood = likelihood
self.layers = layers
def __init__(self,
U,
Y,
likelihood,
kernels,
num_classes=None,
init='random',
X_var=0.01,
**kwargs):
Model.__init__(self, **kwargs)
# U and Y are list of ndarrays
if isinstance(U, np.ndarray):
U = [U]
if isinstance(Y, np.ndarray):
Y = [Y]
self.nSeq = len(Y)
self.kernels = kernels
self.nLayers = len(self.kernels)
self.init = init
self.X_var = X_var
self.likelihood = likelihood
if num_classes is None:
self.num_classes = self.Y[0].shape[1]
else:
self.num_classes = num_classes
self._init_data(U, Y)
self._init_layers()
def __init__(self,
X,
Y,
likelihood,
layers,
minibatch_size=None,
num_samples=1,
**kwargs):
"""
:param X: List of training inputs where each element of the list is a numpy array corresponding to the inputs of one fidelity.
:param Y: List of training targets where each element of the list is a numpy array corresponding to the inputs of one fidelity.
:param likelihood: gpflow likelihood object for use at the final layer
:param layers: List of doubly_stochastic_dgp.layers.Layer objects
:param minibatch_size: Minibatch size if using minibatch trainingz
:param num_samples: Number of samples when propagating predictions through layers
:param kwargs: kwarg inputs to gpflow.models.Model
"""
Model.__init__(self, **kwargs)
self.Y_list = Y
self.X_list = X
self.minibatch_size = minibatch_size
self.num_samples = num_samples
# This allows a training regime where the first layer is trained first by itself, then the subsequent layer
# and so on.
self._train_upto_fidelity = -1
if minibatch_size:
for i, (x, y) in enumerate(zip(X, Y)):
setattr(self, "num_data" + str(i), x.shape[0])
setattr(self, "X" + str(i), Minibatch(x,
minibatch_size,
seed=0))
setattr(self, "Y" + str(i), Minibatch(y,
minibatch_size,
seed=0))
else:
for i, (x, y) in enumerate(zip(X, Y)):
setattr(self, "num_data" + str(i), x.shape[0])
setattr(self, "X" + str(i), DataHolder(x))
setattr(self, "Y" + str(i), DataHolder(y))
self.num_layers = len(layers)
self.layers = ParamList(layers)
self.likelihood = BroadcastingLikelihood(likelihood)
def __init__(self, X, Y, likelihood, layers,
minibatch_size=None,
num_samples=1):
Model.__init__(self)
self.num_samples = num_samples
self.num_data = X.shape[0]
if minibatch_size:
self.X = Minibatch(X, minibatch_size, seed=0)
self.Y = Minibatch(Y, minibatch_size, seed=0)
else:
self.X = DataHolder(X)
self.Y = DataHolder(Y)
self.likelihood = BroadcastingLikelihood(likelihood)
self.layers = ParamList(layers)
def __init__(self,
X,
Y,
time_vec,
likelihood,
layers,
minibatch_size=100,
num_samples=1,
num_data=None,
wfunc='exp',
**kwargs):
Model.__init__(self, **kwargs)
self.num_samples = num_samples
print(np.ndim(X))
if np.ndim(X) == 2:
self.num_data = num_data or X.shape[0]
self.X = wMinibatch(X,
time_vec,
batch_size=minibatch_size,
seed=0,
wfunc=wfunc)
self.Y = wMinibatch(Y,
time_vec,
batch_size=minibatch_size,
seed=0,
wfunc=wfunc)
else:
self.num_data = num_data or X.shape[1]
self.X = wpMinibatch(X,
time_vec,
batch_size=minibatch_size,
seed=0,
wfunc=wfunc)
self.Y = wpMinibatch(Y,
time_vec,
batch_size=minibatch_size,
seed=0,
wfunc=wfunc)
self.m = 4
self.likelihood = BroadcastingLikelihood(likelihood)
self.layers = ParamList(layers)
def __init__(self, X, Y, Z, kernels, likelihood,
num_outputs=None,
mean_function=Zero(), # the final layer mean function
**kwargs):
Model.__init__(self)
num_outputs = num_outputs or Y.shape[1]
# init the layers
layers = []
# inner layers
X_running, Z_running = X.copy(), Z.copy()
for kern_in, kern_out in zip(kernels[:-1], kernels[1:]):
dim_in = kern_in.input_dim
dim_out = kern_out.input_dim
if dim_in == dim_out:
mf = Identity()
else:
if dim_in > dim_out: # stepping down, use the pca projection
_, _, V = np.linalg.svd(X_running, full_matrices=False)
W = V[:dim_out, :].T
else: # pad with zeros
zeros = np.zeros((dim_in, dim_out - dim_in))
W = np.concatenate([np.eye(dim_in), zeros], 1)
mf = Linear(W)
mf.set_trainable(False)
layers.append(SVGP_Layer(kern_in, Z_running, dim_out, mf))
if dim_in != dim_out:
Z_running = Z_running.dot(W)
X_running = X_running.dot(W)
# final layer
layers.append(SVGP_Layer(kernels[-1], Z_running, num_outputs, mean_function))
DGP_Base.__init__(self, X, Y, likelihood, layers, **kwargs)
def __init__(self,
X,
Y,
likelihood,
layers,
minibatch_size=None,
num_samples=1,
num_data=None,
div_weights=None,
**kwargs):
Model.__init__(self, **kwargs)
self.num_samples = num_samples
self.num_data = num_data or X.shape[0]
if minibatch_size:
self.X = Minibatch(X, minibatch_size, seed=0)
self.Y = Minibatch(Y, minibatch_size, seed=0)
else:
self.X = DataHolder(X)
self.Y = DataHolder(Y)
self.likelihood = BroadcastingLikelihood(likelihood)
self.layers = ParamList(layers)
"""CHANGES START"""
"""Weights for the uncertainty quantifiers (per layer)"""
if div_weights is None:
div_weights = [1.0] * len(
layers) #multiply by 1, i.e. don't change
elif type(div_weights) == list and len(div_weights) != len(layers):
print(
"WARNING! You specified a list of weights for the " +
"uncertainty quantifiers, but your DGP has more/less layers " +
"than the number of weights you specified! " +
"We set all weights to 1.0")
div_weights = [1.0] * len(layers)
elif type(div_weights) == list and len(div_weights) == len(layers):
div_weights = div_weights
"""Distribute the weights into the layers"""
for layer, weight in zip(layers, div_weights):
layer.set_weight(weight)
"""CHANGES EEND"""
def __init__(self, X, Y, Z, kernels, likelihood,
num_outputs=None,
mean_function=Zero(), # the final layer mean function
**kwargs):
Model.__init__(self)
num_outputs = num_outputs or Y.shape[1]
# init the layers
layers = []
# inner layers
X_running, Z_running = X.copy(), Z.copy()
for kern_in, kern_out in zip(kernels[:-1], kernels[1:]):
dim_in = kern_in.input_dim
dim_out = kern_out.input_dim
mf = Zero() # Added to compare with DGP EP MCM
# Inducing points for layer
if Z.shape[1] > dim_in: # Reduce Z by doing PCA
_, _, V = np.linalg.svd(X, full_matrices=False) # V -> (D,D) Matrix
Z_kern = Z.dot(V[:dim_in, :].T)
elif Z.shape[1] < dim_in: # Increase Z by doing tile
_, _, V = np.linalg.svd(X, full_matrices=False) # V -> (D,D) Matrix
first_pca = Z.dot(V[0, :].T) # First Principal component
Z_kern = np.tile(first_pca[:, None], (1, dim_in))
else: # same dimension
Z_kern = Z.copy()
layers.append(SVGP_Layer(kern_in, Z_kern, dim_out, mf))
# print("{} \n{}\n".format(Z_kern.shape, Z_kern[0:3, 0:3]))
# Final Layer
mf = Zero() # Added to compare with DGP EP MCM
dim_in = kernels[-1].input_dim
# Inducing points for layer
if Z.shape[1] > dim_in: # Reduce Z by doing PCA
_, _, V = np.linalg.svd(X, full_matrices=False) # V -> (D,D) Matrix
Z_kern = Z.dot(V[:dim_in, :].T)
elif Z.shape[1] < dim_in: # Increase Z by doing tile
_, _, V = np.linalg.svd(X, full_matrices=False) # V -> (D,D) Matrix
first_pca = Z.dot(V[0, :].T) # First Principal component
Z_kern = np.tile(first_pca[:, None], (1, dim_in))
else: # same dimension
Z_kern = Z.copy()
# print("{} \n{}\n".format(Z_kern.shape, Z_kern[0:3, 0:3]))
layers.append(SVGP_Layer(kernels[-1], Z_kern, num_outputs, mean_function))
"""
for kern_in, kern_out in zip(kernels[:-1], kernels[1:]):
dim_in = kern_in.input_dim
dim_out = kern_out.input_dim
if dim_in == dim_out:
mf = Identity()
else: # stepping down, use the pca projection
_, _, V = np.linalg.svd(X_running, full_matrices=False) # V -> (D,D) Matrix
W = V[:dim_out, :].T
mf = Linear(W)
mf.set_trainable(False)
# Z_kern = Z_running[:, 0:dim_in]
# print("{} \n{}\n".format(Z_kern.shape, Z_kern[0:3, 0:3]))
layers.append(SVGP_Layer(kern_in, Z_running, dim_out, mf))
if dim_in != dim_out:
Z_running = Z_running.dot(W)
X_running = X_running.dot(W)
"""
# final layer
# Z_kern = Z_running[:, 0:kernels[-1].input_dim]
# print("{} \n{}\n".format(Z_kern.shape, Z_kern[0:3, 0:3]))
# layers.append(SVGP_Layer(kernels[-1], Z_running, num_outputs, mean_function))
DGP_Base.__init__(self, X, Y, likelihood, layers, **kwargs)
def __init__(
self,
X,
Y,
Z,
kernels,
likelihood,
num_outputs=None,
mean_function=Zero(), # the final layer mean function
**kwargs):
Model.__init__(self)
num_outputs = num_outputs or Y.shape[1]
# init the layers
layers = []
# inner layers
X_running, Z_running = X.copy(), Z.copy()
for kern_in, kern_out in zip(kernels[:-1], kernels[1:]):
if isinstance(kern_in, Conv):
dim_in = kern_in.basekern.input_dim
else:
dim_in = kern_in.input_dim
'''
if isinstance(kern_out,Conv):
dim_out = kern_out.basekern.input_dim
else:
dim_out = kern_out.input_dim
'''
dim_out = kern_out.input_dim
if dim_in == dim_out:
mf = Identity()
else: # stepping down, use the pca projection
_, _, V = np.linalg.svd(X_running, full_matrices=False)
W = V[:dim_out, :].T
b = np.zeros(1, dtype=np.float32)
mf = Linear(W, b)
mf.set_trainable(False)
if isinstance(kern_in, Conv):
Z_patch = np.unique(kern_in.compute_patches(Z_running).reshape(
-1, kern_in.patch_len),
axis=0)
Z_patch = Z_patch[np.random.permutation(
(len(Z_patch)))[:Z_running.shape[0]], :]
layers.append(svconvgp(kern_in, Z_patch, dim_out, mf))
else:
layers.append(SVGP_Layer(kern_in, Z_running, dim_out, mf))
if dim_in != dim_out:
Z_running = Z_running.dot(W)
X_running = X_running.dot(W)
# final layer
if isinstance(kernels[-1], Conv):
Z_patch = np.unique(kernels[-1].compute_patches(Z_running).reshape(
-1, kernels[-1].patch_len),
axis=0)
Z_patch = Z_patch[np.random.permutation(
(len(Z_patch)))[:Z_running.shape[0]], :]
layers.append(
svconvgp(kernels[-1], Z_patch, num_outputs, mean_function))
else:
layers.append(
SVGP_Layer(kernels[-1], Z_running, num_outputs, mean_function))
DGP_Base.__init__(self, X, Y, likelihood, layers, **kwargs)
def __init__(self,
datasets=[],
inducing_locations=[],
kernels=[],
noise_sigmas=[],
minibatch_sizes=[],
mixing_weight=None,
parent_mixtures=None,
masks=None,
num_samples=1,
**kwargs):
"""
datasets: an array of arrays [X_a, Y_a] ordered by 'trust', ie datasets[0] is the most reliable
inducing_points_locations: an array of inducing locations for each of the datasets
kernels: an array of kernels for each of the datasets
noise_sigmas: an array of noise_sigmas for each of the datasets
mixing_weight (MR_Mixing_Weight): an object that will combine the predictions from each of the local experts
parent_mixtures: an array of parent mixture models
"""
Model.__init__(self, **kwargs)
self.dataset_sizes = []
for d in datasets:
self.dataset_sizes.append(d[0].shape[0])
self.num_datasets = len(datasets)
self.X = []
self.Y = []
self.Z = inducing_locations
self.masks = masks
self.MASKS = []
self.kernels = kernels
self.noise_sigmas = noise_sigmas
self.num_samples = num_samples
#gpflow models are Parameterized objects
print(parent_mixtures)
self.parent_mixtures = ParamList(
parent_mixtures) if parent_mixtures is not None else None
self.mixing_weight = mixing_weight
minibatch = False
for i, d in enumerate(datasets):
#TODO: can we just wrap with a ParamList?
if minibatch:
_x = Minibatch(d[0], batch_size=minibatch_sizes[i], seed=0)
_y = Minibatch(d[1], batch_size=minibatch_sizes[i], seed=0)
else:
_x = DataHolder(d[0])
_y = DataHolder(d[1])
#Check we have some masks
if self.masks:
#Check if we have a mask for this dataset
_mask = None
if self.masks[i] is not None:
if minibatch:
_mask = Minibatch(self.masks[i],
batch_size=minibatch_sizes[0],
seed=0)
else:
_mask = DataHolder(self.masks[i])
#make it so GPFlow can find _x, _y
setattr(self, 'x_{i}'.format(i=i), _x)
setattr(self, 'y_{i}'.format(i=i), _y)
if self.masks:
setattr(self, 'mask_{i}'.format(i=i), _mask)
#save references
self.X.append(self.__dict__['x_{i}'.format(i=i)])
self.Y.append(self.__dict__['y_{i}'.format(i=i)])
if self.masks:
self.MASKS.append(self.__dict__['mask_{i}'.format(i=i)])
self.setup()