def test_default_dtype(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dscalar()
high = tensor.dscalar()
# Should not silently downcast from low and high
out0 = random.uniform(low=low, high=high, size=(42, ))
assert out0.dtype == "float64"
f0 = function([low, high], out0)
val0 = f0(-2.1, 3.1)
assert val0.dtype == "float64"
# Should downcast, since asked explicitly
out1 = random.uniform(low=low, high=high, size=(42, ), dtype="float32")
assert out1.dtype == "float32"
f1 = function([low, high], out1)
val1 = f1(-1.1, 1.1)
assert val1.dtype == "float32"
# Should use floatX
lowf = tensor.fscalar()
highf = tensor.fscalar()
outf = random.uniform(low=lowf, high=highf, size=(42, ))
assert outf.dtype == config.floatX
ff = function([lowf, highf], outf)
valf = ff(np.float32(-0.1), np.float32(0.3))
assert valf.dtype == config.floatX
def test_uniform_vector(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dvector()
high = tensor.dvector()
out = random.uniform(low=low, high=high)
assert out.ndim == 1
f = function([low, high], out)
low_val = [0.1, 0.2, 0.3]
high_val = [1.1, 2.2, 3.3]
seed_gen = np.random.RandomState(utt.fetch_seed())
numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30)))
# Arguments of size (3,)
val0 = f(low_val, high_val)
numpy_val0 = numpy_rng.uniform(low=low_val, high=high_val)
assert np.all(val0 == numpy_val0)
# arguments of size (2,)
val1 = f(low_val[:-1], high_val[:-1])
numpy_val1 = numpy_rng.uniform(low=low_val[:-1], high=high_val[:-1])
assert np.all(val1 == numpy_val1)
# Specifying the size explicitly
g = function([low, high], random.uniform(low=low,
high=high,
size=(3, )))
val2 = g(low_val, high_val)
numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30)))
numpy_val2 = numpy_rng.uniform(low=low_val, high=high_val, size=(3, ))
assert np.all(val2 == numpy_val2)
with pytest.raises(ValueError):
g(low_val[:-1], high_val[:-1])
def test_default_dtype(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dscalar()
high = tensor.dscalar()
# Should not silently downcast from low and high
out0 = random.uniform(low=low, high=high, size=(42,))
assert out0.dtype == 'float64'
f0 = function([low, high], out0)
val0 = f0(-2.1, 3.1)
assert val0.dtype == 'float64'
# Should downcast, since asked explicitly
out1 = random.uniform(low=low, high=high, size=(42,), dtype='float32')
assert out1.dtype == 'float32'
f1 = function([low, high], out1)
val1 = f1(-1.1, 1.1)
assert val1.dtype == 'float32'
# Should use floatX
lowf = tensor.fscalar()
highf = tensor.fscalar()
outf = random.uniform(low=lowf, high=highf, size=(42,))
assert outf.dtype == config.floatX
ff = function([lowf, highf], outf)
valf = ff(numpy.float32(-0.1), numpy.float32(0.3))
assert valf.dtype == config.floatX
def test_uniform_vector(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dvector()
high = tensor.dvector()
out = random.uniform(low=low, high=high)
assert out.ndim == 1
f = function([low, high], out)
low_val = [.1, .2, .3]
high_val = [1.1, 2.2, 3.3]
seed_gen = numpy.random.RandomState(utt.fetch_seed())
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
# Arguments of size (3,)
val0 = f(low_val, high_val)
numpy_val0 = numpy_rng.uniform(low=low_val, high=high_val)
print('THEANO', val0)
print('NUMPY', numpy_val0)
assert numpy.all(val0 == numpy_val0)
# arguments of size (2,)
val1 = f(low_val[:-1], high_val[:-1])
numpy_val1 = numpy_rng.uniform(low=low_val[:-1], high=high_val[:-1])
print('THEANO', val1)
print('NUMPY', numpy_val1)
assert numpy.all(val1 == numpy_val1)
# Specifying the size explicitly
g = function([low, high], random.uniform(low=low, high=high, size=(3,)))
val2 = g(low_val, high_val)
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
numpy_val2 = numpy_rng.uniform(low=low_val, high=high_val, size=(3,))
assert numpy.all(val2 == numpy_val2)
self.assertRaises(ValueError, g, low_val[:-1], high_val[:-1])
def test_uniform_vector(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dvector()
high = tensor.dvector()
out = random.uniform(low=low, high=high)
assert out.ndim == 1
f = function([low, high], out)
low_val = [.1, .2, .3]
high_val = [1.1, 2.2, 3.3]
seed_gen = numpy.random.RandomState(utt.fetch_seed())
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
# Arguments of size (3,)
val0 = f(low_val, high_val)
numpy_val0 = numpy_rng.uniform(low=low_val, high=high_val)
print 'THEANO', val0
print 'NUMPY', numpy_val0
assert numpy.all(val0 == numpy_val0)
# arguments of size (2,)
val1 = f(low_val[:-1], high_val[:-1])
numpy_val1 = numpy_rng.uniform(low=low_val[:-1], high=high_val[:-1])
print 'THEANO', val1
print 'NUMPY', numpy_val1
assert numpy.all(val1 == numpy_val1)
# Specifying the size explicitly
g = function([low, high], random.uniform(low=low, high=high, size=(3,)))
val2 = g(low_val, high_val)
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
numpy_val2 = numpy_rng.uniform(low=low_val, high=high_val, size=(3,))
assert numpy.all(val2 == numpy_val2)
self.assertRaises(ValueError, g, low_val[:-1], high_val[:-1])
def __init__(self, neurons, dimensions, count = 1, max_rate = (200, 300),
intercept = (-1.0, 1.0), t_ref = 0.002, t_rc = 0.02, seed = None,
type = 'lif', dt = 0.001, encoders = None, name = None, address = "localhost"):
self.seed = seed
self.neurons = neurons
self.dimensions = dimensions
self.count = count
self.name = name
self.address = address
self.ticker_conn = None
# create the neurons
# TODO: handle different neuron types, which may have different parameters to pass in
self.neuron = neuron.names[type]((count, self.neurons), t_rc = t_rc, t_ref = t_ref, dt = dt)
# compute alpha and bias
srng = RandomStreams(seed=seed)
max_rates = srng.uniform([neurons], low=max_rate[0], high=max_rate[1])
threshold = srng.uniform([neurons], low=intercept[0], high=intercept[1])
alpha, self.bias = theano.function([], self.neuron.make_alpha_bias(max_rates,threshold))()
self.bias = self.bias.astype('float32')
# compute encoders
self.encoders = make_encoders(neurons, dimensions, srng, encoders=encoders)
self.encoders = (self.encoders.T * alpha).T
# make default origin
self.origin = dict(X=origin.Origin(self))
self.accumulator = {}
def test_vector_arguments(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dvector()
out = random.uniform(low=low, high=1)
assert out.ndim == 1
f = function([low], out)
seed_gen = np.random.RandomState(utt.fetch_seed())
numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30)))
val0 = f([-5, 0.5, 0, 1])
val1 = f([0.9])
numpy_val0 = numpy_rng.uniform(low=[-5, 0.5, 0, 1], high=1)
numpy_val1 = numpy_rng.uniform(low=[0.9], high=1)
assert np.all(val0 == numpy_val0)
assert np.all(val1 == numpy_val1)
high = tensor.vector()
outb = random.uniform(low=low, high=high)
assert outb.ndim == 1
fb = function([low, high], outb)
numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30)))
val0b = fb([-4.0, -2], [-1, 0])
val1b = fb([-4.0], [-1])
numpy_val0b = numpy_rng.uniform(low=[-4.0, -2], high=[-1, 0])
numpy_val1b = numpy_rng.uniform(low=[-4.0], high=[-1])
assert np.all(val0b == numpy_val0b)
assert np.all(val1b == numpy_val1b)
with pytest.raises(ValueError):
fb([-4.0, -2], [-1, 0, 1])
# TODO: do we want that?
# with pytest.raises(ValueError):
# fb([-4., -2], [-1])
size = tensor.lvector()
outc = random.uniform(low=low, high=high, size=size, ndim=1)
fc = function([low, high, size], outc)
numpy_rng = np.random.RandomState(int(seed_gen.randint(2**30)))
val0c = fc([-4.0, -2], [-1, 0], [2])
val1c = fc([-4.0], [-1], [1])
numpy_val0c = numpy_rng.uniform(low=[-4.0, -2], high=[-1, 0])
numpy_val1c = numpy_rng.uniform(low=[-4.0], high=[-1])
assert np.all(val0c == numpy_val0c)
assert np.all(val1c == numpy_val1c)
with pytest.raises(ValueError):
fc([-4.0, -2], [-1, 0], [1, 2])
with pytest.raises(ValueError):
fc([-4.0, -2], [-1, 0], [2, 1])
with pytest.raises(ValueError):
fc([-4.0, -2], [-1, 0], [1])
with pytest.raises(ValueError):
fc([-4.0, -2], [-1], [1])
def __init__(
self,
neurons,
dimensions,
tau_ref=0.002,
tau_rc=0.02,
max_rate=(200, 300),
intercept=(-1.0, 1.0),
radius=1.0,
encoders=None,
seed=None,
neuron_type="lif",
dt=0.001,
array_count=1,
):
"""Create an population of neurons with NEF parameters on top
:param int neurons: number of neurons in this population
:param int dimensions: number of dimensions in signal these neurons represent
:param float tau_ref: refractory period of neurons in this population
:param float tau_rc: RC constant
:param tuple max_rate: lower and upper bounds on randomly generate firing rates for neurons in this population
:param tuple intercept: lower and upper bounds on randomly generated x offset
:param float radius:
:param list encoders: set of possible preferred directions of neurons
:param int seed: seed value for random number generator
:param string neuron_type: type of neuron model to use, options = {'lif'}
:param float dt: time step of neurons during update step
:param int array_count: number of sub-populations - for network arrays
"""
self.seed = seed
self.neurons = neurons
self.dimensions = dimensions
self.array_count = array_count
# create the neurons
# TODO: handle different neuron types, which may have different parameters to pass in
self.neuron = neuron.names[neuron_type]((array_count, self.neurons), tau_rc=tau_rc, tau_ref=tau_ref, dt=dt)
# compute alpha and bias
srng = RandomStreams(seed=seed) # set up theano random number generator
max_rates = srng.uniform([neurons], low=max_rate[0], high=max_rate[1])
threshold = srng.uniform([neurons], low=intercept[0], high=intercept[1])
alpha, self.bias = theano.function([], self.neuron.make_alpha_bias(max_rates, threshold))()
self.bias = self.bias.astype("float32") # force to 32 bit for consistency / speed
# compute encoders
self.encoders = make_encoders(neurons, dimensions, srng, encoders=encoders)
self.encoders = (self.encoders.T * alpha).T # combine encoders and gain for simplification
# make default origin
self.origin = dict(X=origin.Origin(self)) # creates origin with identity function
self.accumulator = {} # dictionary of accumulators tracking terminations with different pstc values
def test_binomial_from_uniform_cpu(self):
#Test using numpy
rng = np.random.RandomState(42)
probs = rng.rand(10)
seed = 1337
nb_samples = 1000000
rng = np.random.RandomState(seed)
success1 = np.zeros(len(probs))
for i in range(nb_samples):
success1 += rng.binomial(n=1, p=probs)
rng = np.random.RandomState(seed)
success2 = np.zeros(len(probs))
for i in range(nb_samples):
success2 += (rng.rand(len(probs)) < probs).astype('int')
success1 = success1 / nb_samples
success2 = success2 / nb_samples
assert_array_almost_equal(success1, success2)
#Test using Theano's default RandomStreams
theano_rng = RandomStreams(1337)
rng_bin = theano_rng.binomial(size=probs.shape,
n=1,
p=probs,
dtype=theano.config.floatX)
success1 = np.zeros(len(probs))
for i in range(nb_samples):
success1 += rng_bin.eval()
theano_rng = RandomStreams(1337)
rng_bin = theano_rng.uniform(size=probs.shape,
dtype=theano.config.floatX) < probs
success2 = np.zeros(len(probs))
for i in range(nb_samples):
success2 += rng_bin.eval()
assert_array_almost_equal(success1 / nb_samples, success2 / nb_samples)
#Test using Theano's sandbox MRG RandomStreams
theano_rng = MRG_RandomStreams(1337)
success1 = theano_rng.binomial(size=probs.shape,
n=1,
p=probs,
dtype=theano.config.floatX)
theano_rng = MRG_RandomStreams(1337)
success2 = theano_rng.uniform(size=probs.shape,
dtype=theano.config.floatX) < probs
assert_array_equal(success1.eval(), success2.eval())
def test_mixed_shape(self):
# Test when the provided shape is a tuple of ints and scalar vars
random = RandomStreams(utt.fetch_seed())
shape0 = tensor.lscalar()
shape = (shape0, 3)
f = function([shape0], random.uniform(size=shape, ndim=2))
assert f(2).shape == (2, 3)
assert f(8).shape == (8, 3)
g = function([shape0], random.uniform(size=shape))
assert g(2).shape == (2, 3)
assert g(8).shape == (8, 3)
def test_mixed_shape(self):
# Test when the provided shape is a tuple of ints and scalar vars
random = RandomStreams(utt.fetch_seed())
shape0 = tensor.lscalar()
shape = (shape0, 3)
f = function([shape0], random.uniform(size=shape, ndim=2))
assert f(2).shape == (2, 3)
assert f(8).shape == (8, 3)
g = function([shape0], random.uniform(size=shape))
assert g(2).shape == (2, 3)
assert g(8).shape == (8, 3)
def common_init(self, mr, vr, sr, di, ce, node_id):
"""
Initialization function used by the base class and subclasses.
"""
self.MEANRATE = mr
self.VARRATE = vr
self.STARVRATE = sr
self.DIMS = di
self.CENTS = ce
self.ID = node_id
srng = RandomStreams(seed=100)
rv_u = srng.uniform((self.CENTS, self.DIMS))
f = function([], rv_u)
self.mean = 2*f()
#print self.mean
var1 = T.dscalar('var1')
var2 = T.dmatrix('var2')
var3 = T.mul
self.var = theanoScaMatMul(0.001,np.ones((self.CENTS, self.DIMS)))
self.starv = np.ones((self.CENTS, 1))
self.belief = np.zeros((1, self.CENTS))
self.children = []
self.last = np.zeros((1, self.DIMS))
self.whitening = False
def test_mixed_shape_bcastable(self):
# Test when the provided shape is a tuple of ints and scalar vars
random = RandomStreams(utt.fetch_seed())
shape0 = tensor.lscalar()
shape = (shape0, 1)
u = random.uniform(size=shape, ndim=2)
assert u.broadcastable == (False, True)
f = function([shape0], u)
assert f(2).shape == (2, 1)
assert f(8).shape == (8, 1)
v = random.uniform(size=shape)
assert v.broadcastable == (False, True)
g = function([shape0], v)
assert g(2).shape == (2, 1)
assert g(8).shape == (8, 1)
def __init__(self, latent_dim, hidden_dim, exploration_probability, clip_value, value_decay, data,
batch_size, exploration_decay_rate):
self.latent_dim = latent_dim
self.words = data["words"]
self.depth = 1 + max(len(w) for w in self.words)
depth = self.depth
self.hidden_dim = hidden_dim
self.characters = data["characters"]
self.charset = data["charset"]
self.charmap = data["charmap"]
self.wordcount = len(self.words)
self.charcount = len(self.charset)
self.generator = Generator("generator", latent_dim, depth, self.charcount, hidden_dim, exploration_probability,
exploration_decay_rate)
self.discriminator = Discriminator("discriminator", depth, self.charcount, hidden_dim)
self.clip_value = np.float32(clip_value)
self.value_decay = theano.shared(np.float32(value_decay), "value_decay")
self.batch_size = batch_size
self.word_vectors = np.vstack([self.word_to_vector(word).reshape((1, -1)) for word in self.words]).astype(
np.int32)
xreal = Input((depth,), name="xreal", dtype="int32")
batch_n = T.iscalar("batch_n")
srng = RandomStreams(seed=234)
z = srng.normal(size=(batch_n, latent_dim))
e = srng.uniform(size=(batch_n, depth), low=0, high=1)
ex = srng.random_integers(size=(batch_n, latent_dim), low=0, high=self.charcount)
# z = Input((latent_dim,), name="z", dtype="float32")
# e = Input((depth,), name="e", dtype="float32")
# ex = Input((depth,), name="ex", dtype="int32")
# xreal = T.imatrix("xreal")
# z = T.fmatrix("z")
# e = T.fmatrix("e")
# ex = T.imatrix("ex")
_, xfake = self.generator.policy(z, e, ex)
xfake = theano.gradient.zero_grad(xfake)
# print("xfake: {}, {}".format(xfake, xfake.type))
# print("xreal: {}, {}".format(xreal, xreal.type))
_, yfake = self.discriminator.discriminator(xfake)
_, yreal = self.discriminator.discriminator(xreal)
dloss = T.mean(yfake, axis=None) - T.mean(yreal, axis=None)
dconstraints = {p: ClipConstraint(self.clip_value) for p in self.discriminator.clip_params}
dopt = Adam(1e-4)
dupdates = dopt.get_updates(self.discriminator.params, dconstraints, dloss)
n = z.shape[0]
outputs_info = [T.zeros((n,), dtype='float32')]
yfaker = T.transpose(yfake[:, ::-1], (1, 0))
vtarget, _ = theano.scan(reward_function, outputs_info=outputs_info, sequences=yfaker,
non_sequences=self.value_decay)
vtarget = T.transpose(vtarget, (1, 0))[:, ::-1]
# print("vtarget: {}, {}, {}".format(vtarget, vtarget.ndim, vtarget.type))
_, vpred = self.generator.value(z, xfake)
gloss = T.mean(T.abs_(vtarget - vpred), axis=None)
gopt = Adam(1e-5)
gupdates = gopt.get_updates(self.generator.params, {}, gloss)
self.discriminator_train_function = theano.function([xreal, batch_n], [dloss], updates=dupdates)
self.generator_train_function = theano.function([batch_n], [gloss], updates=gupdates)
self.generator_sample_function = theano.function([batch_n], [xfake])
self.test_function = theano.function([xreal, batch_n], [dloss, gloss])
def prediction(self, h, bias):
srng = RandomStreams(seed=42)
prop, mean_x, mean_y, std_x, std_y, rho, bernoulli = \
self.compute_parameters(h, bias)
mode = T.argmax(srng.multinomial(pvals=prop, dtype=prop.dtype), axis=1)
v = T.arange(0, mean_x.shape[0])
m_x = mean_x[v, mode]
m_y = mean_y[v, mode]
s_x = std_x[v, mode]
s_y = std_y[v, mode]
r = rho[v, mode]
# cov = r * (s_x * s_y)
normal = srng.normal((h.shape[0], 2))
x = normal[:, 0]
y = normal[:, 1]
# x_n = T.shape_padright(s_x * x + cov * y + m_x)
# y_n = T.shape_padright(s_y * y + cov * x + m_y)
x_n = T.shape_padright(m_x + s_x * x)
y_n = T.shape_padright(m_y + s_y * (x * r + y * T.sqrt(1.-r**2)))
uniform = srng.uniform((h.shape[0],))
pin = T.shape_padright(T.cast(bernoulli > uniform, floatX))
return T.concatenate([x_n, y_n, pin], axis=1)
def __init__(self, rng, train_input, test_input, n_in, n_out):
# self.input = input.flatten(2)
self.W = theano.shared(value=numpy.asarray(rng.uniform(
low=-numpy.sqrt(6. / (n_in + n_out)),
high=numpy.sqrt(6. / (n_in + n_out)),
size=(n_in, n_out)),
dtype=theano.config.floatX),
name='W',
borrow=True)
self.b = theano.shared(value=numpy.zeros((n_out, ),
dtype=theano.config.floatX),
name='b',
borrow=True)
p = 0.5
tmp_output = T.nnet.relu(
T.dot(train_input.flatten(2), self.W) + self.b)
srng = RandomStreams(rng.randint(1234))
mask = (srng.uniform(size=tmp_output.shape) < p) / p
self.train_output = tmp_output * mask
self.test_output = T.nnet.relu(
T.dot(test_input.flatten(2), self.W) + self.b)
self.params = [self.W, self.b]
def __init__(self, rng, input, n_in, n_out):
self.W = []
self.b = []
train_output = []
test_output = []
self.params = []
p = 0.5
for i in range(4):
sub_input = input[:, :, i * 18:(i + 1) * 18, :].flatten(2)
self.W.append(
theano.shared(value=numpy.asarray(rng.uniform(
low=-numpy.sqrt(6. / (n_in + n_out)),
high=numpy.sqrt(6. / (n_in + n_out)),
size=(n_in, n_out)),
dtype='float32'),
borrow=True))
self.b.append(
theano.shared(value=numpy.zeros((n_out, ), dtype='float32'),
borrow=True))
self.params.append(self.W[i])
self.params.append(self.b[i])
tmp_output = T.nnet.relu(T.dot(sub_input, self.W[i]) + self.b[i])
srng = RandomStreams(rng.randint(1234))
mask = (srng.uniform(size=tmp_output.shape) < p) / p
train_output.append(tmp_output * mask)
test_output.append(tmp_output)
self.train_output = T.concatenate(train_output, axis=1)
self.test_output = T.concatenate(test_output, axis=1)
def ssgd2(self,
loss,
all_params,
learning_rate=0.01,
chaos_energy=0.01,
alpha=0.9):
from theano.tensor.shared_randomstreams import RandomStreams
chaos_energy = T.constant(chaos_energy, dtype="float32")
alpha = T.constant(alpha, dtype="float32")
learning_rate = T.constant(learning_rate, dtype="float32")
srng = RandomStreams(seed=3)
updates = []
all_grads = T.grad(loss, all_params)
for p, g in zip(all_params, all_grads):
rand_v = (srng.uniform(p.get_value().shape) * 2 - 1) * chaos_energy
g_ratio_vec = g / g.norm(L=2)
ratio_sum = theano.shared(np.ones(np.array(p.get_value().shape),
dtype="float32"),
name="ssgd2_r_sum_%s" % p.name)
abs_ratio_sum = T.abs_(ratio_sum)
updates.append(
(ratio_sum, ratio_sum * alpha + (1 - alpha) * g_ratio_vec))
updates.append(
(p, p - learning_rate * ((abs_ratio_sum) * g +
(1 - abs_ratio_sum) * rand_v)))
return updates
class Depool1DLayer(Layer):
def __init__(self, output_shape, depool_shape=(2,), depooler='random', rng=np.random):
self.depool_shape = depool_shape
self.output_shape = output_shape
self.theano_rng = RandomStreams(rng.randint(2 ** 30))
self.depooler = depooler
self.params = []
def __call__(self, input):
input = input.dimshuffle(0,1,2,'x').repeat(self.depool_shape[0], axis=3)
if self.depooler == 'random':
output_mask = self.theano_rng.uniform(size=(
self.output_shape[0],self.output_shape[1],
self.output_shape[2]//self.depool_shape[0], self.depool_shape[0]),
dtype=theano.config.floatX)
output_mask = T.floor(output_mask / output_mask.max(axis=3).dimshuffle(0,1,2,'x'))
return (output_mask * input).reshape(self.output_shape)
elif self.depooler == 'first':
output_mask_np = np.zeros(self.depool_shape, dtype=theano.config.floatX)
output_mask_np[0] = 1.0
output_mask = theano.shared(mask_np, borrow=True).dimshuffle('x','x','x',0)
return (output_mask * input).reshape(self.output_shape)
else:
return self.depooler(input, axis=3).reshape(self.output_shape)
def prediction(self, h, bias):
srng = RandomStreams(seed=42)
prop, mean_x, mean_y, std_x, std_y, rho, bernoulli = \
self.compute_parameters(h, bias)
mode = T.argmax(srng.multinomial(pvals=prop, dtype=prop.dtype), axis=1)
v = T.arange(0, mean_x.shape[0])
m_x = mean_x[v, mode]
m_y = mean_y[v, mode]
s_x = std_x[v, mode]
s_y = std_y[v, mode]
r = rho[v, mode]
# cov = r * (s_x * s_y)
normal = srng.normal((h.shape[0], 2))
x = normal[:, 0]
y = normal[:, 1]
# x_n = T.shape_padright(s_x * x + cov * y + m_x)
# y_n = T.shape_padright(s_y * y + cov * x + m_y)
x_n = T.shape_padright(m_x + s_x * x)
y_n = T.shape_padright(m_y + s_y * (x * r + y * T.sqrt(1. - r**2)))
uniform = srng.uniform((h.shape[0], ))
pin = T.shape_padright(T.cast(bernoulli > uniform, floatX))
return T.concatenate([x_n, y_n, pin], axis=1)
def dropout(self, rate=0.5, seed=None):
obj = self.copy()
srng = RandomStreams(seed)
obj.out = T.where(srng.uniform(size=obj.out.shape) > rate, obj.out, 0)
return obj
class GP(object):
def __init__(self,size=20,v0=1,v1=1,v2=1,r=1,time_step=10,alpha=2):
self.lsize = size
self.time = time_step
self.dtype = theano.config.floatX
self.v0 = theano.shared(np.cast[dtype](v0),name="v0")
self.v1 = theano.shared(np.cast[dtype](v1),name="v1")
self.v2 = theano.shared(np.cast[dtype](v2),name="v2")
self.alpha = theano.shared(np.cast[dtype](alpha),name="alpha")
self.gamma = theano.shared(np.cast[dtype](r),name="r")
self.srng = RandomStreams(seed=234)
self.DiffM = theano.shared(self.create_mat(self.time,self.lsize),name="diffM")
self.params = [self.v0,self.gamma,self.alpha]
self.sample = self.return_output(self.DiffM)
##This takes the difference matrix and samples from the GP and returns a signal
def return_output(self,Dif):
#Dif is theano.Tensor.matrix type
Frac = Dif/self.gamma
Cov = self.v0*T.pow(Frac,self.alpha)
L = sin.cholesky(T.exp(-Cov))
eps = self.srng.uniform((self.time,self.lsize))
return T.dot(L,eps)
##This converts the noise signal into the basioc matrix required for Covariance calculation
def create_mat(self,time,size):
#noise is numpy type
diffM = np.zeros(shape=(time,size))
for i in range(0,size):
for j in range(0,size):
diffM[i,j] = i-j
return np.abs(diffM)
def test_mixed_shape_bcastable(self):
# Test when the provided shape is a tuple of ints and scalar vars
random = RandomStreams(utt.fetch_seed())
shape0 = tensor.lscalar()
shape = (shape0, 1)
u = random.uniform(size=shape, ndim=2)
assert u.broadcastable == (False, True)
f = function([shape0], u)
assert f(2).shape == (2, 1)
assert f(8).shape == (8, 1)
v = random.uniform(size=shape)
assert v.broadcastable == (False, True)
g = function([shape0], v)
assert g(2).shape == (2, 1)
assert g(8).shape == (8, 1)
class SampleBinomial(Layer):
def __init__(self, from_logits=False, **kwargs):
super(SampleBinomial, self).__init__(**kwargs)
self.from_logits = from_logits
if K.backend() == 'theano':
from theano.sandbox.rng_mrg import MRG_RandomStreams as RandomStreams
self.random = RandomStreams()
elif K.backend() == 'tensorflow':
import tensorflow as tf
else:
raise NotImplementedError
def call(self, x, mask=None):
if K.backend() == 'theano':
s = self.random.uniform(x.shape, low=0, high=1)
elif K.backend() == 'tensorflow':
import tensorflow as tf
s = tf.random_uniform(tf.shape(x), minval=0, maxval=1)
else:
raise NotImplementedError
if self.from_logits:
# TODO: there might be more direct way from logits
return K.cast(K.sigmoid(x) > s, K.floatx())
else:
return K.cast(x > s, K.floatx())
def __init__(self, rng, train_input, test_input, n_in, n_out):
# self.input = input.flatten(2)
self.W = theano.shared(
value=numpy.asarray(
rng.uniform(
low=-numpy.sqrt(6. / (n_in + n_out)),
high=numpy.sqrt(6. / (n_in + n_out)),
size=(n_in, n_out)
),
dtype=theano.config.floatX
),
name='W',
borrow=True
)
self.b = theano.shared(
value=numpy.zeros((n_out,), dtype=theano.config.floatX),
name='b',
borrow=True
)
p = 0.5
tmp_output = T.nnet.relu(T.dot(train_input.flatten(2), self.W) + self.b)
srng = RandomStreams(rng.randint(1234))
mask = (srng.uniform(size=tmp_output.shape) < p)/p
self.train_output = tmp_output * mask
self.test_output = T.nnet.relu(T.dot(test_input.flatten(2), self.W) + self.b)
self.params = [self.W, self.b]
def test_default_updates(self):
# Basic case: default_updates
random_a = RandomStreams(utt.fetch_seed())
out_a = random_a.uniform((2, 2))
fn_a = function([], out_a)
fn_a_val0 = fn_a()
fn_a_val1 = fn_a()
assert not numpy.all(fn_a_val0 == fn_a_val1)
nearly_zeros = function([], out_a + out_a - 2 * out_a)
assert numpy.all(abs(nearly_zeros()) < 1e-5)
# Explicit updates #1
random_b = RandomStreams(utt.fetch_seed())
out_b = random_b.uniform((2, 2))
fn_b = function([], out_b, updates=random_b.updates())
fn_b_val0 = fn_b()
fn_b_val1 = fn_b()
assert numpy.all(fn_b_val0 == fn_a_val0)
assert numpy.all(fn_b_val1 == fn_a_val1)
# Explicit updates #2
random_c = RandomStreams(utt.fetch_seed())
out_c = random_c.uniform((2, 2))
fn_c = function([], out_c, updates=[out_c.update])
fn_c_val0 = fn_c()
fn_c_val1 = fn_c()
assert numpy.all(fn_c_val0 == fn_a_val0)
assert numpy.all(fn_c_val1 == fn_a_val1)
# No updates at all
random_d = RandomStreams(utt.fetch_seed())
out_d = random_d.uniform((2, 2))
fn_d = function([], out_d, no_default_updates=True)
fn_d_val0 = fn_d()
fn_d_val1 = fn_d()
assert numpy.all(fn_d_val0 == fn_a_val0)
assert numpy.all(fn_d_val1 == fn_d_val0)
# No updates for out
random_e = RandomStreams(utt.fetch_seed())
out_e = random_e.uniform((2, 2))
fn_e = function([], out_e, no_default_updates=[out_e.rng])
fn_e_val0 = fn_e()
fn_e_val1 = fn_e()
assert numpy.all(fn_e_val0 == fn_a_val0)
assert numpy.all(fn_e_val1 == fn_e_val0)
def test_default_updates(self):
# Basic case: default_updates
random_a = RandomStreams(utt.fetch_seed())
out_a = random_a.uniform((2, 2))
fn_a = function([], out_a)
fn_a_val0 = fn_a()
fn_a_val1 = fn_a()
assert not np.all(fn_a_val0 == fn_a_val1)
nearly_zeros = function([], out_a + out_a - 2 * out_a)
assert np.all(abs(nearly_zeros()) < 1e-5)
# Explicit updates #1
random_b = RandomStreams(utt.fetch_seed())
out_b = random_b.uniform((2, 2))
fn_b = function([], out_b, updates=random_b.updates())
fn_b_val0 = fn_b()
fn_b_val1 = fn_b()
assert np.all(fn_b_val0 == fn_a_val0)
assert np.all(fn_b_val1 == fn_a_val1)
# Explicit updates #2
random_c = RandomStreams(utt.fetch_seed())
out_c = random_c.uniform((2, 2))
fn_c = function([], out_c, updates=[out_c.update])
fn_c_val0 = fn_c()
fn_c_val1 = fn_c()
assert np.all(fn_c_val0 == fn_a_val0)
assert np.all(fn_c_val1 == fn_a_val1)
# No updates at all
random_d = RandomStreams(utt.fetch_seed())
out_d = random_d.uniform((2, 2))
fn_d = function([], out_d, no_default_updates=True)
fn_d_val0 = fn_d()
fn_d_val1 = fn_d()
assert np.all(fn_d_val0 == fn_a_val0)
assert np.all(fn_d_val1 == fn_d_val0)
# No updates for out
random_e = RandomStreams(utt.fetch_seed())
out_e = random_e.uniform((2, 2))
fn_e = function([], out_e, no_default_updates=[out_e.rng])
fn_e_val0 = fn_e()
fn_e_val1 = fn_e()
assert np.all(fn_e_val0 == fn_a_val0)
assert np.all(fn_e_val1 == fn_e_val0)
def test_vector_arguments(self):
random = RandomStreams(utt.fetch_seed())
low = tensor.dvector()
out = random.uniform(low=low, high=1)
assert out.ndim == 1
f = function([low], out)
seed_gen = numpy.random.RandomState(utt.fetch_seed())
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
val0 = f([-5, .5, 0, 1])
val1 = f([.9])
numpy_val0 = numpy_rng.uniform(low=[-5, .5, 0, 1], high=1)
numpy_val1 = numpy_rng.uniform(low=[.9], high=1)
assert numpy.all(val0 == numpy_val0)
assert numpy.all(val1 == numpy_val1)
high = tensor.vector()
outb = random.uniform(low=low, high=high)
assert outb.ndim == 1
fb = function([low, high], outb)
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
val0b = fb([-4., -2], [-1, 0])
val1b = fb([-4.], [-1])
numpy_val0b = numpy_rng.uniform(low=[-4., -2], high=[-1, 0])
numpy_val1b = numpy_rng.uniform(low=[-4.], high=[-1])
assert numpy.all(val0b == numpy_val0b)
assert numpy.all(val1b == numpy_val1b)
self.assertRaises(ValueError, fb, [-4., -2], [-1, 0, 1])
# TODO: do we want that?
#self.assertRaises(ValueError, fb, [-4., -2], [-1])
size = tensor.lvector()
outc = random.uniform(low=low, high=high, size=size, ndim=1)
fc = function([low, high, size], outc)
numpy_rng = numpy.random.RandomState(int(seed_gen.randint(2**30)))
val0c = fc([-4., -2], [-1, 0], [2])
val1c = fc([-4.], [-1], [1])
numpy_val0c = numpy_rng.uniform(low=[-4., -2], high=[-1, 0])
numpy_val1c = numpy_rng.uniform(low=[-4.], high=[-1])
assert numpy.all(val0c == numpy_val0c)
assert numpy.all(val1c == numpy_val1c)
self.assertRaises(ValueError, fc, [-4., -2], [-1, 0], [1])
self.assertRaises(ValueError, fc, [-4., -2], [-1, 0], [1, 2])
self.assertRaises(ValueError, fc, [-4., -2], [-1, 0], [2, 1])
self.assertRaises(ValueError, fc, [-4., -2], [-1], [1])
class Pool2DLayer:
def __init__(self,
rng,
input_shape,
pool_shape=(2, 2),
pooler=T.max,
depooler='random'):
self.pool_shape = pool_shape
self.input_shape = input_shape
self.theano_rng = RandomStreams(rng.randint(2**30))
self.pooler = pooler
self.depooler = depooler
self.params = []
def __call__(self, input):
input = input.reshape(
(self.input_shape[0], self.input_shape[1],
self.input_shape[2] // self.pool_shape[0], self.pool_shape[0],
self.input_shape[3] // self.pool_shape[1], self.pool_shape[1]))
input = self.pooler(input, axis=5)
input = self.pooler(input, axis=3)
return input
def inv(self, output):
output = (output.dimshuffle(0, 1, 2, 'x', 3, 'x').repeat(
self.pool_shape[1], axis=5).repeat(self.pool_shape[0], axis=3))
if self.depooler == 'random':
unpooled = (self.input_shape[0], self.input_shape[1],
self.input_shape[2] // self.pool_shape[0],
self.pool_shape[0], self.input_shape[3] //
self.pool_shape[1], self.pool_shape[1])
pooled = (self.input_shape[0], self.input_shape[1],
self.input_shape[2] // self.pool_shape[0], 1,
self.input_shape[3] // self.pool_shape[1], 1)
output_mask = self.theano_rng.uniform(size=unpooled,
dtype=theano.config.floatX)
output_mask = output_mask / output_mask.max(axis=5).max(
axis=3).dimshuffle(0, 1, 2, 'x', 3, 'x')
output_mask = T.floor(output_mask)
return (output_mask * output).reshape(self.input_shape)
else:
output = self.depooler(output, axis=5)
output = self.depooler(output, axis=3)
return output
def load(self, filename):
pass
def save(self, filename):
pass
def __init__(self, input, rescale, recentre):
srng = RandomStreams(seed=234)
self.input = input
dequantize_input = input + srng.uniform(size=input.shape, low=-0.5/255, high=0.5/255)
self.output = rescale * (dequantize_input - recentre)
def test_binomial_from_uniform_cpu(self):
#Test using numpy
rng = np.random.RandomState(42)
probs = rng.rand(10)
seed = 1337
nb_samples = 1000000
rng = np.random.RandomState(seed)
success1 = np.zeros(len(probs))
for i in range(nb_samples):
success1 += rng.binomial(n=1, p=probs)
rng = np.random.RandomState(seed)
success2 = np.zeros(len(probs))
for i in range(nb_samples):
success2 += (rng.rand(len(probs)) < probs).astype('int')
success1 = success1 / nb_samples
success2 = success2 / nb_samples
assert_array_almost_equal(success1, success2)
#Test using Theano's default RandomStreams
theano_rng = RandomStreams(1337)
rng_bin = theano_rng.binomial(size=probs.shape, n=1, p=probs, dtype=theano.config.floatX)
success1 = np.zeros(len(probs))
for i in range(nb_samples):
success1 += rng_bin.eval()
theano_rng = RandomStreams(1337)
rng_bin = theano_rng.uniform(size=probs.shape, dtype=theano.config.floatX) < probs
success2 = np.zeros(len(probs))
for i in range(nb_samples):
success2 += rng_bin.eval()
assert_array_almost_equal(success1/nb_samples, success2/nb_samples)
#Test using Theano's sandbox MRG RandomStreams
theano_rng = MRG_RandomStreams(1337)
success1 = theano_rng.binomial(size=probs.shape, n=1, p=probs, dtype=theano.config.floatX)
theano_rng = MRG_RandomStreams(1337)
success2 = theano_rng.uniform(size=probs.shape, dtype=theano.config.floatX) < probs
assert_array_equal(success1.eval(), success2.eval())
def func_dec(self, xt, htm1, ctm1):
# xt -- embedded world representations
current_att_weight = self.softmax(
theano.dot(
tensor.tanh(
theano.dot(
htm1, self.W_att_target
) + self.scope_att_times_W
),
self.b_att
)
)
#
zt = theano.dot(current_att_weight, self.scope_att)
#
post_transform = self.b_dec + theano.dot(
tensor.concatenate(
[xt, htm1, zt], axis=0
),
self.W_dec
)
gate_input = tensor.nnet.sigmoid(
post_transform[:self.dim_model]
)
gate_forget = tensor.nnet.sigmoid(
post_transform[self.dim_model:2*self.dim_model]
)
gate_output = tensor.nnet.sigmoid(
post_transform[2*self.dim_model:3*self.dim_model]
)
gate_pre_c = tensor.tanh(
post_transform[3*self.dim_model:]
)
ct = gate_forget * ctm1 + gate_input * gate_pre_c
ht = gate_output * tensor.tanh(ct)
'''
Add drop out here, by cha chen
'''
# Set up an random number generator
srng = RandomStreams(seed=0)
# Set up the dropout
windows = srng.uniform((self.dim_model,)) < 0.9
getwins = theano.function([],windows)
winst = getwins()
ht_dropout = ht * winst
# return the dropout version
return ht, ht_dropout, ct, zt
#
# return ht, ct, zt
'''
class Pool1DLayer:
def __init__(self, rng, pool_shape, input_shape, pooler=T.max, depooler='random'):
"""
Allocate a Pool1DLayer with shared variable internal parameters.
:type rng: numpy.random.RandomState
:param rng: a random number generator used to initialize weights
:type pool_shape: tuple or list of length 4
:param pool_shape: (pool height, pool width)
:type input_shape: tuple or list of length 3
:param input_shape: (num input feature maps, number of joints, number of time frames)
:type pooler: theano.tensor method
:param pooler: Type of pooling operation (by deafult max pooling)
:type depooler: string
:param depooler: Type of deepoling operation (random or first)
"""
self.pool_shape = pool_shape
self.input_shape = input_shape
self.output_shape = (input_shape[0], input_shape[1], input_shape[2]//self.pool_shape[0])
self.theano_rng = RandomStreams(rng.randint(2 ** 30))
self.pooler = pooler
self.depooler = depooler
self.params = []
def __call__(self, input):
return self.pooler(input.reshape((
self.input_shape[0], self.input_shape[1],
self.input_shape[2]//self.pool_shape[0],
self.pool_shape[0])), axis=3)
def inv(self, output):
output = output.dimshuffle(0,1,2,'x').repeat(self.pool_shape[0], axis=3)
if self.depooler == 'random':
mask = self.theano_rng.uniform(size=output.shape, dtype=theano.config.floatX)
mask = T.floor(mask / mask.max(axis=3).dimshuffle(0,1,2,'x'))
output = mask * output
elif self.depooler == 'first':
mask_np = np.zeros(self.pool_shape, dtype=theano.config.floatX)
mask_np[0] = 1.0
mask = theano.shared(mask_np, borrow=True).dimshuffle('x','x','x',0)
output = mask * output
else:
output = self.depooler(output, axis=3)
return output.reshape(self.input_shape)
def load(self, filename): pass
def save(self, filename): pass
def test_symbolic_shape(self):
random = RandomStreams(utt.fetch_seed())
shape = tensor.lvector()
f = function([shape], random.uniform(size=shape, ndim=2))
assert f([2, 3]).shape == (2, 3)
assert f([4, 8]).shape == (4, 8)
self.assertRaises(ValueError, f, [4])
self.assertRaises(ValueError, f, [4, 3, 4, 5])
def test_symbolic_shape(self):
random = RandomStreams(utt.fetch_seed())
shape = tensor.lvector()
f = function([shape], random.uniform(size=shape, ndim=2))
assert f([2, 3]).shape == (2, 3)
assert f([4, 8]).shape == (4, 8)
self.assertRaises(ValueError, f, [4])
self.assertRaises(ValueError, f, [4, 3, 4, 5])
class RNN(Model):
def __init__(self, nin, nout, nhid, numpy_rng, scale=1.0):
self.nin = nin
self.nout = nout
self.nhid = nhid
self.numpy_rng = numpy_rng
self.theano_rng = RandomStreams(1)
self.scale = np.float32(scale)
self.inputs = T.fmatrix('inputs')
self.targets = T.imatrix('targets')
self.masks = T.bmatrix('masks')
self.batchsize = self.inputs.shape[0]
self.inputs_frames = self.inputs.reshape((
self.batchsize, self.inputs.shape[1]/nin, nin)).dimshuffle(1,0,2)
self.targets_frames = self.targets.T
self.masks_frames = self.masks.T
self.win = theano.shared(value=self.numpy_rng.normal(
loc=0, scale=0.001, size=(nin, nhid)
).astype(theano.config.floatX), name='win')
self.wrnn = theano.shared(value=self.scale * np.eye(
nhid, dtype=theano.config.floatX), name='wrnn')
self.wout = theano.shared(value=self.numpy_rng.uniform(
low=-0.01, high=0.01, size=(nhid, nout)
).astype(theano.config.floatX), name='wout')
self.bout = theano.shared(value=np.zeros(
nout, dtype=theano.config.floatX), name='bout')
self.params = [self.win, self.wrnn, self.wout, self.bout]
(self.hiddens, self.outputs), self.updates = theano.scan(
fn=self.step, sequences=self.inputs_frames,
outputs_info=[self.theano_rng.uniform(low=0, high=1, size=(
self.batchsize, nhid), dtype=theano.config.floatX), None])
self.probabilities = T.nnet.softmax(self.outputs.reshape((
self.outputs.shape[0] * self.outputs.shape[1],
self.nout)))
self.probabilities = T.clip(self.probabilities, 1e-6, 1-1e-6)
self._stepcosts = T.nnet.categorical_crossentropy(
self.probabilities, self.targets_frames.flatten()).reshape(
self.targets_frames.shape)
self._cost = T.switch(T.gt(self.masks_frames, 0), self._stepcosts, 0).mean()
self._grads = T.grad(self._cost, self.params)
self.get_classifications = theano.function(
[self.inputs], T.argmax(self.probabilities.reshape(self.outputs.shape), axis=2).T)
def step(self, inp_t, h_tm1):
pre_h_t = T.dot(inp_t, self.win) + T.dot(h_tm1, self.wrnn)
h_t = T.switch(pre_h_t > 0, pre_h_t, 0)
o_t = T.dot(h_t, self.wout) + self.bout
return h_t, o_t
def expr(self, model, data, **kwargs):
"""
Overwrites the Cost.expr so we can inject our theano.Op.
"""
space,source = self.get_data_specs(model)
space.validate(data)
#really no point to using these random values. Could be zeros
srng = RandomStreams(seed=234)
return OverwriteOp(self.cost,model)(srng.uniform(low=0.0,high=1000.0,dtype=theano.config.floatX),data)
def __init__(self, neurons, dimensions, tau_ref=0.002, tau_rc=0.02, max_rate=(200,300), intercept=(-1.0,1.0),
radius=1.0, encoders=None, seed=None, neuron_type='lif', dt=0.001, array_size=1, eval_points=None):
"""Create an population of neurons with NEF parameters on top
:param int neurons: number of neurons in this population
:param int dimensions: number of dimensions in signal these neurons represent
:param float tau_ref: refractory period of neurons in this population
:param float tau_rc: RC constant
:param tuple max_rate: lower and upper bounds on randomly generate firing rates for neurons in this population
:param tuple intercept: lower and upper bounds on randomly generated x offset
:param float radius: the range of input values (-radius:radius) this population is sensitive to
:param list encoders: set of possible preferred directions of neurons
:param int seed: seed value for random number generator
:param string neuron_type: type of neuron model to use, options = {'lif'}
:param float dt: time step of neurons during update step
:param int array_size: number of sub-populations - for network arrays
:param list eval_points: specific set of points to optimize decoders over by default for this ensemble
"""
self.seed = seed
self.neurons_num = neurons
self.dimensions = dimensions
self.array_size = array_size
self.radius = radius
self.eval_points = eval_points
# create the neurons
# TODO: handle different neuron types, which may have different parameters to pass in
self.neurons = neuron.names[neuron_type]((array_size, self.neurons_num), tau_rc=tau_rc, tau_ref=tau_ref, dt=dt)
# compute alpha and bias
srng = RandomStreams(seed=seed) # set up theano random number generator
max_rates = srng.uniform([self.neurons_num], low=max_rate[0], high=max_rate[1])
threshold = srng.uniform([self.neurons_num], low=intercept[0], high=intercept[1])
alpha, self.bias = theano.function([], self.neurons.make_alpha_bias(max_rates, threshold))()
self.bias = self.bias.astype('float32') # force to 32 bit for consistency / speed
# compute encoders
self.encoders = make_encoders(self.neurons_num, dimensions, srng, encoders=encoders)
self.encoders = (self.encoders.T * alpha).T # combine encoders and gain for simplification
self.origin = {} # make origin dictionary
self.add_origin('X', func=None, eval_points=self.eval_points) # make default origin
self.accumulators = {} # dictionary of accumulators tracking terminations with different pstc values
def __init__(self, neurons, dimensions, count = 1, max_rate = (200, 300),
intercept = (-1.0, 1.0), t_ref = 0.002, t_rc = 0.02, seed = None,
type='lif', dt=0.001, encoders=None,
is_subensemble=False, name=None, decoders=None, bias=None):
self.name = name
self.seed = seed
self.neurons = neurons
self.dimensions = dimensions
self.count = count
self.accumulator = {}
self.is_subensemble = is_subensemble
# create the neurons
# TODO: handle different neuron types, which may have different parameters to pass in
# The structure of the data contained in self.neuron consists of several variables that are
# arrays of the form
# Array([
# [x_0_0, x_0_1, x_0_2,..., x_0_(neurons - 1)],
# [x_1_0, x_1_1, x_1_2,..., x_1_(neurons - 1)],
# [...],
# [x_(count-1)_0, x_(count-1)_1, x_(count-1)_2,..., x_$count-1)_(neurons - 1)]
# ])
self.neuron = neuron.names[type]((count, self.neurons), t_rc = t_rc, t_ref = t_ref, dt = dt)
if is_subensemble:
self.bias = bias
self.encoders = encoders
else:
# compute alpha and bias
srng = RandomStreams(seed=seed)
max_rates = srng.uniform([neurons], low=max_rate[0], high=max_rate[1])
threshold = srng.uniform([neurons], low=intercept[0], high=intercept[1])
alpha, self.bias = theano.function([], self.neuron.make_alpha_bias(max_rates, threshold))()
self.bias = self.bias.astype('float32')
# compute encoders
self.encoders = make_encoders(neurons, dimensions, srng, encoders=encoders)
self.encoders = (self.encoders.T * alpha).T
# make default origin
self.origin = dict(X=origin.Origin(self, decoder=decoders))
def __init__(
self,
encoder,
decoder,
opt,
tau0,
taurate,
taumin,
K=10, # number of classes
N=30, # number of categorical distributions
eps=1e-9):
srng = RandomStreams(123)
# input image x (shape=(batch_size,784))
x = T.imatrix(name="x")
# variational posterior q(y|x), i.e. the encoder (shape=(batch_size,200))
# unnormalized logits for N separate K-categorical distributions (shape=(batch_size*N,K))
logits_y = T.reshape(encoder.call(x), (-1, K))
q_y = T.nnet.softmax(logits_y)
log_q_y = T.log(q_y + eps)
# temperature
iteration = theano.shared(0, name='iteration')
iter_updates = [(iteration, iteration + 1)]
tau = T.max(T.stack((taumin, tau0 * T.exp(-taurate * iteration))))
# sample and reshape back (shape=(batch_size,N,K))
# set hard=True for ST Gumbel-Softmax
yraw = gumbel_softmax(logits_y, tau, hard=False, srng=srng)
yflat = T.reshape(yraw, (-1, N * K))
# generative model p(x|y), i.e. the decoder (shape=(batch_size,200))
logits_x = decoder.call(yflat)
# (shape=(batch_size,784))
p_x = T.nnet.sigmoid(logits_x)
p_xt = bernoulli_px(p_x, x)
kl = T.mean(T.sum(q_y * (log_q_y - T.log(1.0 / K)), axis=1), axis=0)
elbo = T.mean(-T.sum(T.log(eps + p_xt), axis=1), axis=0)
loss = elbo - kl
params = encoder.params + decoder.params
updates = opt.get_updates(loss, params)
self.train_function = theano.function([x], [elbo, kl, loss, tau],
updates=updates + iter_updates)
ymax = T.argmax(logits_y, axis=1)
yonehot = T.reshape(tensor_one_hot(ymax, K), (-1, N * K))
pxval = T.nnet.sigmoid(decoder.call(yonehot))
pxtval = bernoulli_px(pxval, x)
valloss = T.mean(-T.sum(T.log(eps + pxtval), axis=1), axis=0)
self.validate_function = theano.function([x],
[elbo, kl, loss, valloss])
rnd = srng.uniform(low=0., high=1., size=x.shape)
sampled = T.gt(pxval, rnd)
self.autoencode_function = theano.function([x], sampled)
self.tau_function = theano.function([], tau)
self.weights = params + opt.weights + [iteration]
def random_stuff():
rng = RandomStreams(seed = None)
a = rng.uniform((10, 10))
b = rng.normal((10, 1))
tdbplot(a, 'a', ImagePlot(cmap = 'jet'))
tdbplot(b, 'b', HistogramPlot(edges = np.linspace(-5, 5, 20)))
c = a+b
return c
def test_ndim(self):
"""Test that the behaviour of 'ndim' optional parameter"""
# 'ndim' is an optional integer parameter, specifying the length
# of the 'shape', passed as a keyword argument.
# ndim not specified, OK
random = RandomStreams(utt.fetch_seed())
fn = function([], random.uniform((2, 2)))
# ndim specified, consistent with shape, OK
random2 = RandomStreams(utt.fetch_seed())
fn2 = function([], random2.uniform((2, 2), ndim=2))
val1 = fn()
val2 = fn2()
assert numpy.all(val1 == val2)
# ndim specified, inconsistent with shape, should raise ValueError
random3 = RandomStreams(utt.fetch_seed())
self.assertRaises(ValueError, random3.uniform, (2, 2), ndim=1)
class Pool2DLayer:
def __init__(self, rng, input_shape, pool_shape=(2,2), pooler=T.max, depooler='random'):
self.pool_shape = pool_shape
self.input_shape = input_shape
self.theano_rng = RandomStreams(rng.randint(2 ** 30))
self.pooler = pooler
self.depooler = depooler
self.params = []
def __call__(self, input):
input = input.reshape((
self.input_shape[0],self.input_shape[1],
self.input_shape[2]//self.pool_shape[0],self.pool_shape[0],
self.input_shape[3]//self.pool_shape[1],self.pool_shape[1]))
input = self.pooler(input, axis=5)
input = self.pooler(input, axis=3)
return input
def inv(self, output):
output = (output.dimshuffle(0,1,2,'x',3,'x')
.repeat(self.pool_shape[1], axis=5)
.repeat(self.pool_shape[0], axis=3))
if self.depooler == 'random':
unpooled = (
self.input_shape[0], self.input_shape[1],
self.input_shape[2]//self.pool_shape[0], self.pool_shape[0],
self.input_shape[3]//self.pool_shape[1], self.pool_shape[1])
pooled = (
self.input_shape[0], self.input_shape[1],
self.input_shape[2]//self.pool_shape[0], 1,
self.input_shape[3]//self.pool_shape[1], 1)
output_mask = self.theano_rng.uniform(size=unpooled, dtype=theano.config.floatX)
output_mask = output_mask / output_mask.max(axis=5).max(axis=3).dimshuffle(0,1,2,'x',3,'x')
output_mask = T.floor(output_mask)
return (output_mask * output).reshape(self.input_shape)
else:
output = self.depooler(output, axis=5)
output = self.depooler(output, axis=3)
return output
def load(self, filename): pass
def save(self, filename): pass
def test_ndim(self):
"""Test that the behaviour of 'ndim' optional parameter"""
# 'ndim' is an optional integer parameter, specifying the length
# of the 'shape', passed as a keyword argument.
# ndim not specified, OK
random = RandomStreams(utt.fetch_seed())
fn = function([], random.uniform((2, 2)))
# ndim specified, consistent with shape, OK
random2 = RandomStreams(utt.fetch_seed())
fn2 = function([], random2.uniform((2, 2), ndim=2))
val1 = fn()
val2 = fn2()
assert numpy.all(val1 == val2)
# ndim specified, inconsistent with shape, should raise ValueError
random3 = RandomStreams(utt.fetch_seed())
self.assertRaises(ValueError, random3.uniform, (2, 2), ndim=1)
def random_stuff():
rng = RandomStreams(seed=None)
a = rng.uniform((10, 10))
b = rng.normal((10, 1))
tdbplot(a, 'a', ImagePlot(cmap='jet'))
tdbplot(b, 'b', HistogramPlot(edges=np.linspace(-5, 5, 20)))
c = a + b
return c
def get_gradients(self, model, data, ** kwargs):
"""
Overwrites the Cost.get_gradients so we can inject our theano.Op
This will do a separate call back for each model.param
Consider rewriting your model to have one param
"""
srng = RandomStreams(seed=232)
params = list(model.get_params())
grads = [OverwriteOp(self.grad,model)(srng.uniform(size=i.shape,dtype=theano.config.floatX),data) for i in params]
gradients = OrderedDict(izip(params, grads))
updates = OrderedDict()
return gradients, updates
def predictor_step(self, stack, cursors, stack_mask, stack_current_value_tm1, stack_prev_value_tm1, input_current_value_tm1, policy_tm1, policy_calculated_tm1, data, mask):
value_masks = self.get_value_masks(cursors, stack_mask)
values = self.get_values(stack, data, value_masks)
reduced = self.calc_reduced_value(values)
action, policy = self.calc_action(values)
srng = RandomStreams()
select_random_action = TS.le(srng.uniform((self.batch_size,)), self.random_action_prob)
random_action = TS.ge(srng.uniform((self.batch_size,)), 0.5)
action = (1-select_random_action)*action + select_random_action*random_action
action, no_action, policy_calculated = self.apply_border_conditions(cursors, stack_mask, mask, action)
stack_reduce_result = self.calc_stack_after_reduce(stack, value_masks, reduced)
stack_shift_result = self.calc_stack_after_shift(stack, value_masks, values)
# Итоговое новое состояние агента в зависимости от выполняемых операций (SHIFT или REDUCE)
do_shift = TS.eq(action, 1)
do_shift = do_shift.dimshuffle([0,'x'])
do_shift = TS.extra_ops.repeat(do_shift, self.max_len, axis=1)
new_cursors = K.switch(do_shift, cursors + value_masks['input_next'], cursors)
new_stack_mask = K.switch(do_shift, stack_mask + value_masks['stack_next'], stack_mask - value_masks['stack_current'])
do_shift = do_shift.dimshuffle([0, 1,'x'])
do_shift = TS.extra_ops.repeat(do_shift, 2*self.hidden_dim, axis=2)
new_stack = K.switch(do_shift, stack_shift_result, stack_reduce_result)
no_action_mask = no_action.dimshuffle([0,'x'])
no_action_mask = TS.extra_ops.repeat(no_action_mask, self.max_len, axis=1)
new_cursors = K.switch(no_action_mask, cursors, new_cursors)
new_stack_mask = K.switch(no_action_mask, stack_mask, new_stack_mask)
no_action_mask = no_action_mask.dimshuffle([0, 1,'x'])
no_action_mask = TS.extra_ops.repeat(no_action_mask, 2*self.hidden_dim, axis=2)
new_stack = K.switch(no_action_mask, stack, new_stack)
stack_current_value = values['stack_current'][:, self.hidden_dim:]
stack_prev_value = values['stack_prev'][:, self.hidden_dim:]
input_current_value = values['input_current'][:, self.hidden_dim:]
return new_stack, new_cursors, new_stack_mask, stack_current_value, stack_prev_value, input_current_value, policy, policy_calculated
def test_tutorial(self):
srng = RandomStreams(seed=234)
rv_u = srng.uniform((2, 2))
rv_n = srng.normal((2, 2))
f = function([], rv_u)
g = function([], rv_n, no_default_updates=True) # Not updating rv_n.rng
nearly_zeros = function([], rv_u + rv_u - 2 * rv_u)
assert numpy.all(f() != f())
assert numpy.all(g() == g())
assert numpy.all(abs(nearly_zeros()) < 1e-5)
assert isinstance(rv_u.rng.get_value(borrow=True), numpy.random.RandomState)
def __call__(self, x, **kwargs):
if self.p > 0:
from theano.tensor.shared_randomstreams import RandomStreams
if x.dtype.startswith('int'):
#make one hot version of x
shape = list(x.shape)+[self.dim]
mesh = T.mgrid[0:x.shape[0],0:x.shape[1]]
i,j = mesh[0], mesh[1]
x_one_hot = T.set_subtensor(T.zeros(shape)[i,j,x], 1)
x = x_one_hot
rng = RandomStreams(seed=self.seed)
I = rng.uniform((x.shape[0],x.shape[1])) < self.p
x = T.set_subtensor(x[I.nonzero()], 1.0/self.dim)
return x
def test_tutorial(self):
srng = RandomStreams(seed=234)
rv_u = srng.uniform((2, 2))
rv_n = srng.normal((2, 2))
f = function([], rv_u)
g = function([], rv_n,
no_default_updates=True) # Not updating rv_n.rng
nearly_zeros = function([], rv_u + rv_u - 2 * rv_u)
assert np.all(f() != f())
assert np.all(g() == g())
assert np.all(abs(nearly_zeros()) < 1e-5)
assert isinstance(rv_u.rng.get_value(borrow=True),
np.random.RandomState)
def test_uniform(self):
"""Test that RandomStreams.uniform generates the same results as numpy"""
# Check over two calls to see if the random state is correctly updated.
random = RandomStreams(utt.fetch_seed())
fn = function([], random.uniform((2, 2), -1, 1))
fn_val0 = fn()
fn_val1 = fn()
rng_seed = numpy.random.RandomState(utt.fetch_seed()).randint(2**30)
rng = numpy.random.RandomState(int(rng_seed)) # int() is for 32bit
numpy_val0 = rng.uniform(-1, 1, size=(2, 2))
numpy_val1 = rng.uniform(-1, 1, size=(2, 2))
assert numpy.allclose(fn_val0, numpy_val0)
assert numpy.allclose(fn_val1, numpy_val1)
def test_uniform(self):
# Test that RandomStreams.uniform generates the same results as numpy
# Check over two calls to see if the random state is correctly updated.
random = RandomStreams(utt.fetch_seed())
fn = function([], random.uniform((2, 2), -1, 1))
fn_val0 = fn()
fn_val1 = fn()
rng_seed = np.random.RandomState(utt.fetch_seed()).randint(2**30)
rng = np.random.RandomState(int(rng_seed)) # int() is for 32bit
numpy_val0 = rng.uniform(-1, 1, size=(2, 2))
numpy_val1 = rng.uniform(-1, 1, size=(2, 2))
assert np.allclose(fn_val0, numpy_val0)
assert np.allclose(fn_val1, numpy_val1)
class ColorReweightLayer(Layer):
def __init__(self, incoming, **kwargs):
self.rng = RandomStreams()
super(ColorReweightLayer, self).__init__(incoming, **kwargs)
# self.low = low
# self.high = high
def get_output_for(self, input, **kwargs):
weights = self.rng.uniform(size=(input.shape[0], input.shape[1]))
#weights = 3 * weights / T.sum(weights, axis=1).reshape((input.shape[0], 1))
output = input * weights.reshape(
(input.shape[0], input.shape[1], 1, 1))
# output = T.minimum(output, self.high)
# output = T.maximum(output, self.low)
return output
def test_setitem(self):
random = RandomStreams(234)
out = random.uniform((2, 2))
fn = function([], out, updates=random.updates())
random.seed(888)
rng = numpy.random.RandomState(utt.fetch_seed())
random[out.rng] = numpy.random.RandomState(utt.fetch_seed())
fn_val0 = fn()
fn_val1 = fn()
numpy_val0 = rng.uniform(size=(2, 2))
numpy_val1 = rng.uniform(size=(2, 2))
assert numpy.allclose(fn_val0, numpy_val0)
assert numpy.allclose(fn_val1, numpy_val1)
def generateData(dim,num):
names=[]
data={}
regions={}
targets={}
srng = RandomStreams()
rv_u = srng.uniform(dim)
rv_u = T.mul(rv_u,50.0)
rv_u = T.sub(rv_u,25.0)
f = function([], rv_u)
for i in range(num):
name=str(i)
names.append(name)
data[name]=f()
regions[name]=(0,0,dim[0]-1,dim[1]-1)
#targets[name]=np.sum(data[name],dtype=np.float32)
targets[name]=np.amax(data[name])
return names,data,regions,targets
def ssgd2(self, loss, all_params, learning_rate=0.01, chaos_energy=0.01, alpha=0.9):
from theano.tensor.shared_randomstreams import RandomStreams
chaos_energy = T.constant(chaos_energy, dtype="float32")
alpha = T.constant(alpha, dtype="float32")
learning_rate = T.constant(learning_rate, dtype="float32")
srng = RandomStreams(seed=3)
updates = []
all_grads = T.grad(loss, all_params)
for p, g in zip(all_params, all_grads):
rand_v = (srng.uniform(p.get_value().shape)*2 - 1) * chaos_energy
g_ratio_vec = g / g.norm(L=2)
ratio_sum = theano.shared(np.ones(np.array(p.get_value().shape), dtype="float32"), name="ssgd2_r_sum_%s" % p.name)
abs_ratio_sum = T.abs_(ratio_sum)
updates.append((ratio_sum, ratio_sum * alpha + (1 - alpha ) * g_ratio_vec))
updates.append((p, p - learning_rate*((abs_ratio_sum)*g + (1-abs_ratio_sum)*rand_v)))
return updates
