def init_tparams(params,
given_tparams=None,
given_dup_tparams=None,
use_mv=False):
"""Initialize Theano shared variables according to the initial parameters"""
tparams = OrderedDict() if given_tparams is None else given_tparams
dup_tparams = OrderedDict(
) if given_dup_tparams is None else given_dup_tparams
if not use_mv:
for kk, pp in params.iteritems():
tparams[kk] = theano.shared(params[kk], name=kk)
else:
try:
from multiverso.theano_ext import sharedvar
except ImportError:
from ..multiverso_.theano_ext import sharedvar
for kk, pp in params.iteritems():
if any(kk.startswith(var) for var in dup_shared_var_list):
tparams[kk] = theano.shared(params[kk], name=kk)
dup_tparams[kk] = sharedvar.mv_shared(value=np.ones(dup_size) *
params[kk][0],
name=kk,
borrow=False)
else:
tparams[kk] = sharedvar.mv_shared(value=params[kk],
name=kk,
borrow=False)
return tparams, dup_tparams
def __init__(self, input, n_in, n_out):
""" Initialize the parameters of the logistic regression
:type input: theano.tensor.TensorType
:param input: symbolic variable that describes the input of the
architecture (one minibatch)
:type n_in: int
:param n_in: number of input units, the dimension of the space in
which the datapoints lie
:type n_out: int
:param n_out: number of output units, the dimension of the space in
which the labels lie
"""
# start-snippet-1
# initialize with 0 the weights W as a matrix of shape (n_in, n_out)
# MULTIVERSO: replace the shared variable with mv_shared
self.W = sharedvar.mv_shared(value=numpy.zeros(
(n_in, n_out), dtype=theano.config.floatX),
name='W',
borrow=True)
# initialize the biases b as a vector of n_out 0s
# MULTIVERSO: replace the shared variable with mv_shared
self.b = sharedvar.mv_shared(value=numpy.zeros(
(n_out, ), dtype=theano.config.floatX),
name='b',
borrow=True)
# symbolic expression for computing the matrix of class-membership
# probabilities
# Where:
# W is a matrix where column-k represent the separation hyperplane for
# class-k
# x is a matrix where row-j represents input training sample-j
# b is a vector where element-k represent the free parameter of
# hyperplane-k
self.p_y_given_x = T.nnet.softmax(T.dot(input, self.W) + self.b)
# symbolic description of how to compute prediction as class whose
# probability is maximal
self.y_pred = T.argmax(self.p_y_given_x, axis=1)
# end-snippet-1
# parameters of the model
self.params = [self.W, self.b]
# keep track of model input
self.input = input
def momentum(cost, params, learning_rate, momentum):
grads = theano.grad(cost, params)
updates = []
for p, g in zip(params, grads):
# MULTIVERSO: relace the shared variable with mv_shared
mparam_i = sharedvar.mv_shared(np.zeros(p.get_value().shape, dtype=theano.config.floatX))
v = momentum * mparam_i - learning_rate * g
updates.append((mparam_i, v))
updates.append((p, p + v))
return updates
def momentum(cost, params, learning_rate, momentum):
grads = theano.grad(cost, params)
updates = []
for p, g in zip(params, grads):
# MULTIVERSO: relace the shared variable with mv_shared
mparam_i = sharedvar.mv_shared(
np.zeros(p.get_value().shape, dtype=theano.config.floatX))
v = momentum * mparam_i - learning_rate * g
updates.append((mparam_i, v))
updates.append((p, p + v))
return updates
def _test_sharedvar(self, row, col):
W = sharedvar.mv_shared(value=np.zeros((row, col),
dtype=theano.config.floatX),
name='W',
borrow=True)
delta = np.array(range(1, row * col + 1),
dtype=theano.config.floatX).reshape((row, col))
train_model = theano.function([], updates=[(W, W + delta)])
mv.barrier()
for i in xrange(100):
train_model()
train_model()
sharedvar.sync_all_mv_shared_vars()
mv.barrier()
# to get the newest value, we must sync again
sharedvar.sync_all_mv_shared_vars()
for j, actual in enumerate(W.get_value().reshape(-1)):
self.assertEqual((j + 1) * (i + 1) * 2 * mv.workers_num(),
actual)
mv.barrier()
def _test_sharedvar(self, row, col):
W = sharedvar.mv_shared(
value=np.zeros(
(row, col),
dtype=theano.config.floatX
),
name='W',
borrow=True
)
delta = np.array(range(1, row * col + 1),
dtype=theano.config.floatX).reshape((row, col))
train_model = theano.function([], updates=[(W, W + delta)])
mv.barrier()
for i in xrange(100):
train_model()
train_model()
sharedvar.sync_all_mv_shared_vars()
mv.barrier()
# to get the newest value, we must sync again
sharedvar.sync_all_mv_shared_vars()
for j, actual in enumerate(W.get_value().reshape(-1)):
self.assertEqual((j + 1) * (i + 1) * 2 * mv.workers_num(), actual)
mv.barrier()
def init_weights(shape, name):
# MULTIVERSO: relace the shared variable with mv_shared
return sharedvar.mv_shared(floatX(np.random.randn(*shape) * 0.1),
name=name)
def init_weights(shape, name):
# MULTIVERSO: relace the shared variable with mv_shared
return sharedvar.mv_shared(floatX(np.random.randn(*shape) * 0.1), name=name)
