def _make_args(self, X, Z, imp_weight=None):
batch_size = getattr(self, 'batch_size', None)
if batch_size is None:
X, Z = cast_array_to_local_type(X), cast_array_to_local_type(Z)
if imp_weight is not None:
imp_weight = cast_array_to_local_type(imp_weight)
data = itertools.repeat([X, Z, imp_weight])
else:
data = itertools.repeat([X, Z])
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
if imp_weight is not None:
data = iter_minibatches([X, Z, imp_weight], self.batch_size,
list(self.sample_dim) + [self.sample_dim[0]])
data = ((cast_array_to_local_type(x),
cast_array_to_local_type(z),
cast_array_to_local_type(w)) for x, z, w in data)
else:
data = iter_minibatches([X, Z], self.batch_size,
self.sample_dim)
data = ((cast_array_to_local_type(x),
cast_array_to_local_type(z)) for x, z in data)
args = ((i, {}) for i in data)
return args
def _make_args(self, X, Z, imp_weight=None):
batch_size = getattr(self, 'batch_size', None)
if batch_size is None:
X, Z = cast_array_to_local_type(X), cast_array_to_local_type(Z)
if imp_weight is not None:
imp_weight = cast_array_to_local_type(imp_weight)
data = itertools.repeat([X, Z, imp_weight])
else:
data = itertools.repeat([X, Z])
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
if imp_weight is not None:
data = iter_minibatches([X, Z, imp_weight], self.batch_size,
list(self.sample_dim) +
[self.sample_dim[0]])
data = ((cast_array_to_local_type(x),
cast_array_to_local_type(z),
cast_array_to_local_type(w)) for x, z, w in data)
else:
data = iter_minibatches([X, Z], self.batch_size,
self.sample_dim)
data = ((cast_array_to_local_type(x),
cast_array_to_local_type(z)) for x, z in data)
args = ((i, {}) for i in data)
return args
def _make_args(self, X, W=None):
batch_size = getattr(self, 'batch_size', None)
use_imp_weight = W is not None
if self.use_imp_weight != use_imp_weight:
raise ValueError('need to give ``W`` in accordinace to '
'``self.imp_weight``')
item = [X, W] if use_imp_weight else [X]
# If importance weights are used, we need to append the sample
# dimensionality of it, which will be the same as for that data.
sample_dim = list(self.sample_dim)
if use_imp_weight:
sample_dim.append(sample_dim[0])
if batch_size is None:
data = itertools.repeat(item)
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
data = iter_minibatches(item, self.batch_size, sample_dim)
if use_imp_weight:
data = ((cast_array_to_local_type(x), cast_array_to_local_type(w))
for x, w in data)
else:
data = ((cast_array_to_local_type(x),)
for x, in data)
args = ((i, {}) for i in data)
return args
def _make_args(self, X, W=None):
batch_size = getattr(self, 'batch_size', None)
use_imp_weight = W is not None
if self.use_imp_weight != use_imp_weight:
raise ValueError('need to give ``W`` in accordinace to '
'``self.imp_weight``')
item = [X, W] if use_imp_weight else [X]
# If importance weights are used, we need to append the sample
# dimensionality of it, which will be the same as for that data.
sample_dim = list(self.sample_dim)
if use_imp_weight:
sample_dim.append(sample_dim[0])
if batch_size is None:
data = itertools.repeat(item)
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
data = iter_minibatches(item, self.batch_size, sample_dim)
if use_imp_weight:
data = ((cast_array_to_local_type(x), cast_array_to_local_type(w))
for x, w in data)
else:
data = ((cast_array_to_local_type(x), ) for x, in data)
args = ((i, {}) for i in data)
return args
def _make_args(self, X):
batch_size = getattr(self, 'batch_size', None)
if batch_size is None:
data = itertools.repeat([X])
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
data = iter_minibatches([X], self.batch_size, self.sample_dim)
args = ((i, {}) for i in data)
return args
def __call__(self, f_score, *data):
""""Return the score of the data."""
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
this_samples = int(batch[0].shape[self.sample_dims[0]])
score += f_score(*batch) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def _make_args(self, X):
batch_size = getattr(self, 'batch_size', None)
if batch_size is None:
data = itertools.repeat([X])
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
data = iter_minibatches([X], self.batch_size, self.sample_dim)
args = ((i, {}) for i in data)
return args
def __call__(self, f_score, *data):
""""Return the score of the data."""
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
this_samples = batch[0].shape[self.sample_dims[0]]
score += f_score(*batch) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
x = batch[0]
z = batch[1]
this_samples = int(x.shape[0])
score += f_score(x, z) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, f_score, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
score = 0.
seen_samples = 0.
for batch in batches:
x = batch[0]
z = batch[1]
this_samples = int(x.shape[0])
score += f_score(x, z) * this_samples
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, predict_f, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
seen_samples = 0.
score = 0.
for batch in batches:
x, z = batch
y = predict_f(x)
this_samples = int(y.shape[1])
errs = (y.argmax(axis=2) != z.argmax(axis=2)).sum()
score += errs
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def __call__(self, predict_f, *data):
batches = iter_minibatches(data, self.max_samples, self.sample_dims, 1)
seen_samples = 0.
score = 0.
for batch in batches:
x, z = batch
y = predict_f(x)
this_samples = int(y.shape[1])
errs = (y.argmax(axis=2) != z.argmax(axis=2)).sum()
score += errs
seen_samples += this_samples
return ma.scalar(score / seen_samples)
def _make_args(self, X, Z):
batch_size = getattr(self, 'batch_size', None)
if batch_size is None:
X, Z = cast_array_to_local_type(X), cast_array_to_local_type(Z)
data = itertools.repeat([X, Z])
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
data = iter_minibatches([X, Z], self.batch_size, self.sample_dim)
data = ((cast_array_to_local_type(x), cast_array_to_local_type(z))
for x, z in data)
args = ((i, {}) for i in data)
return args
def _make_args(self, X, Z):
batch_size = getattr(self, 'batch_size', None)
if batch_size is None:
X, Z = cast_array_to_local_type(X), cast_array_to_local_type(Z)
data = itertools.repeat([X, Z])
elif batch_size < 1:
raise ValueError('need strictly positive batch size')
else:
data = iter_minibatches([X, Z], self.batch_size, self.sample_dim)
data = ((cast_array_to_local_type(x), cast_array_to_local_type(z))
for x, z in data)
args = ((i, {}) for i in data)
return args
def __init__(self, X, y, inputs, cov, batch_size, hermgauss_deg=100):
"""
:param X: data points (possibly a batch for some methods)
:param y: target values
:param inputs: inducing inputs (positions)
:param cov: covariance function
:param batch_size: size of the mini batch on which the ELBO is computed
:param hermgauss_deg: degree of Gauss-Hermite approximation
"""
ELBO.__init__(self, X, y, 'svi')
self.cov = cov
self.inputs = inputs
m = inputs.shape[0]
self.mu = np.zeros((m, 1))
self.SigmaL = np.eye(m)
self.batches = (i
for i in iter_minibatches([X, y], batch_size, [0, 0]))
self.gauss_hermite = hermgauss(hermgauss_deg)
def climin_wrapper(oracle, w0, train_points, train_targets, options, method='AdaDelta'):
default_options = {'maxiter': 1000, 'print_freq':1, 'verbose': False, 'g_tol': 1e-5,
'batch_size': 10, 'step_rate': 0.1}
if not options is None:
default_options.update(options)
if 'print_freq' in options.keys():
default_options['verbose'] = True
options = default_options
w = w0.copy()
data = ((i, {}) for i in iter_minibatches([train_points, train_targets], options['batch_size'], [1, 0]))
if method == 'AdaDelta':
opt = climin.Adadelta(wrt=w, fprime=oracle, args=data, step_rate=options['step_rate'])
elif method == 'SG':
opt = climin.GradientDescent(wrt=w, fprime=oracle, args=data, step_rate=options['step_rate'])
else:
raise ValueError('Unknown optimizer')
w_lst = [w.copy()]
time_lst = [0.]
start = time.time()
n_epochs = options['maxiter']
n_iterations = int(n_epochs * train_targets.size / options['batch_size'])
print_freq = int(options['print_freq'] * train_targets.size / options['batch_size'])
if options['verbose']:
print('Using ' + method + ' optimizer')
for info in opt:
i = info['n_iter']
if i > n_iterations:
break
if not (i % print_freq) and options['verbose']:
grad = info['gradient']
print("Iteration ", int(i * options['batch_size'] / train_targets.size), ":")
print("\tGradient norm", np.linalg.norm(grad))
if not i % int(train_targets.size / options['batch_size']):
w_lst.append(w.copy())
time_lst.append(time.time() - start)
return w.copy(), w_lst, time_lst
def _climin_setup(self, train_x, train_y):
"""
The functions sets up the climin minibatches protocol given the data to train on.
:param train_x: the data to train on
:param train_y: the labels for the data
:return: params: a flat placeholder for the model parameters
args: a list of arguments passed to the climin optimizer at each training step
"""
# initialize a flat parameters placeholder for climin
# randomize the parameters, but keep variance small enough (0.01) to promote faster learning
params = np.zeros(
self.feature_dim * self.output_dim + self.output_dim) # 0.01 * random.randn(self.feature_dim * self.output_dim + self.output_dim)
# pass self during the training to update the parameters of the linear regression model object itself
# and reuse the theano definitions from the constructor
minibatches = cli_util.iter_minibatches([train_x, train_y], self.batch_size, [0, 0])
# as climin arguments, pass the mini_batch_x, mini_batch_y and the model itself
args = (([minibatch[0], minibatch[1], self], {}) for minibatch in minibatches)
return params, args
def _climin_setup(self, train_x, train_y):
"""
The functions sets up the climin minibatches protocol given the data to train on.
:param train_x: the data to train on
:param train_y: the labels for the data
:return: params: a flat placeholder for the model parameters
args: a list of arguments passed to the climin optimizer at each training step
"""
# initialize a flat parameters placeholder for climin
# randomize the parameters, but keep variance small enough (0.01) to promote faster learning
size = 0
for layer in self.layers:
size = size + layer.in_dim * layer.out_dim + layer.out_dim
params = 0.01 * random.randn(size)
# define the additional arguments for each train iteration
minibatches = cli_util.iter_minibatches([train_x, train_y], self.batch_size, [0, 0])
# as climin arguments, pass the mini_batch_x, mini_batch_y and the model itself
args = (([minibatch[0], minibatch[1], self], {}) for minibatch in minibatches)
return params, args
def climin_wrapper(oracle,
w0,
train_points,
train_targets,
options,
method='AdaDelta'):
default_options = {
'maxiter': 1000,
'print_freq': 1,
'verbose': False,
'g_tol': 1e-5,
'batch_size': 10,
'step_rate': 0.1
}
if not options is None:
default_options.update(options)
if 'print_freq' in options.keys():
default_options['verbose'] = True
options = default_options
w = w0.copy()
data = ((i, {}) for i in iter_minibatches([train_points, train_targets],
options['batch_size'], [1, 0]))
if method == 'AdaDelta':
opt = climin.Adadelta(wrt=w,
fprime=oracle,
args=data,
step_rate=options['step_rate'])
elif method == 'SG':
opt = climin.GradientDescent(wrt=w,
fprime=oracle,
args=data,
step_rate=options['step_rate'])
else:
raise ValueError('Unknown optimizer')
w_lst = [w.copy()]
time_lst = [0.]
start = time.time()
n_epochs = options['maxiter']
n_iterations = int(n_epochs * train_targets.size / options['batch_size'])
print_freq = int(options['print_freq'] * train_targets.size /
options['batch_size'])
if options['verbose']:
print('Using ' + method + ' optimizer')
for info in opt:
i = info['n_iter']
if i > n_iterations:
break
if not (i % print_freq) and options['verbose']:
grad = info['gradient']
print("Iteration ",
int(i * options['batch_size'] / train_targets.size), ":")
print("\tGradient norm", np.linalg.norm(grad))
if not i % int(train_targets.size / options['batch_size']):
w_lst.append(w.copy())
time_lst.append(time.time() - start)
return w.copy(), w_lst, time_lst
