def test_func(func):
y = func(x)
yT = func.T(func(x))
self.assertEquals(K.eval(y).shape, K.get_shape(y))
self.assertEquals(K.eval(yT).shape, (25, 8, 12))
self.assertEquals(K.eval(yT).shape, K.get_shape(yT))
def _apply(self, X, state=None, memory=None):
# time_major: The shape format of the `inputs` and `outputs` Tensors.
# If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
# If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
# ====== create attention if necessary ====== #
cell = self.cell
if self.bidirectional:
cell_bw = self.cell_bw
# create attention cell
if self.attention:
if not hasattr(self, "_cell_with_attention"):
self._cell_with_attention = self.__attention_creator(
cell, X=X, memory=memory)
cell = self._cell_with_attention
# bidirectional attention
if self.bidirectional:
if not hasattr(self, "_cell_with_attention_bw"):
self._cell_with_attention_bw = self.__attention_creator(
cell_bw, X=X, memory=memory)
cell_bw = self._cell_with_attention_bw
# ====== calling rnn_warpper ====== #
## Bidirectional
if self.bidirectional:
rnn_func = rnn.bidirectional_dynamic_rnn if self.dynamic \
else rnn.static_bidirectional_rnn
state_fw, state_bw = None, None
if isinstance(state, (tuple, list)):
state_fw = state[0]
if len(state) > 1:
state_bw = state[1]
else:
state_fw = state
outputs = rnn_func(cell_fw=cell,
cell_bw=cell_bw,
inputs=X,
initial_state_fw=state_fw,
initial_state_bw=state_bw,
dtype=X.dtype.base_dtype)
## Unidirectional
else:
rnn_func = rnn.dynamic_rnn if self.dynamic else rnn.static_rnn
outputs = rnn_func(cell,
inputs=X,
initial_state=state,
dtype=X.dtype.base_dtype)
# ====== initialize cell ====== #
if not self._is_initialized_variables:
# initialize only once, everytime you call this, the values of
# variables changed
K.eval(tf.variables_initializer(self.variables))
self._is_initialized_variables = True
_infer_variable_role(self.variables)
# ====== return ====== #
if self.bidirectional: # concat outputs
outputs = (tf.concat(outputs[0], axis=-1), outputs[1])
if not self.return_states:
return outputs[0]
return outputs
def test_save_cudnn_rnn(self):
np.random.seed(5218)
X = K.variable(np.random.rand(25, 12, 8))
num_layers = 2
num_gates = 'lstm'
skip_input = False
is_bidirectional = False
path = '/tmp/rnn'
weights, biases = K.init_rnn(input_dim=8, hidden_dim=18,
b_init=init_ops.random_normal_initializer(),
num_layers=num_layers, num_gates=num_gates,
skip_input=skip_input,
is_bidirectional=is_bidirectional)
rnn = N.CudnnRNN(num_units=18,
W_init=weights, b_init=biases,
rnn_mode=num_gates, num_layers=num_layers,
skip_input=skip_input, is_bidirectional=is_bidirectional,
return_states=False,
dropout=0., name="CudnnRNNTest")
y = rnn(X)
K.initialize_all_variables()
y = K.eval(y)
N.serialize(nnops=rnn, path=path, binary_output=True, override=True)
test_script = r"""
from __future__ import print_function, division, absolute_import
import os
os.environ['ODIN'] = 'gpu,float32,seed=5218'
import pickle
import numpy as np
import tensorflow as tf
from tensorflow.python.ops import init_ops
from odin.config import randint
from odin import backend as K, nnet as N
np.random.seed(5218)
X = K.variable(np.random.rand(25, 12, 8))
rnn = N.deserialize("%s", force_restore_vars=True)
y = rnn(X)
K.initialize_all_variables()
y = K.eval(y)
print(len(rnn.variables),
sum(np.sum(K.eval(i)) for i in rnn.variables
if K.role.has_roles(i, K.role.Weight)),
sum(np.sum(K.eval(i)) for i in rnn.variables
if K.role.has_roles(i, K.role.Bias)),
y.sum(),
(y**2).sum())
""" % path
outputs = run_script(test_script)[1]
num_variables, w, b, s1, s2 = outputs.split(' ')
assert int(num_variables) == len(rnn.variables)
assert np.allclose(float(w),
sum(np.sum(K.eval(i)) for i in rnn.variables
if K.role.has_roles(i, K.role.Weight)))
assert np.allclose(float(b),
sum(np.sum(K.eval(i)) for i in rnn.variables
if K.role.has_roles(i, K.role.Bias)))
assert np.allclose(float(s1), y.sum())
assert np.allclose(float(s2), (y**2).sum())
def test_variable_creation(self):
np.random.seed(5218)
# ====== create by numpy array ====== #
tmp = np.random.rand(12, 8).astype('float32')
K.variable(value=tmp, dtype='float32', name='x', initialize=True)
self.assertTrue(np.all(K.eval(K.variable(name='x')) == tmp))
# ====== create by Variable name ====== #
K.variable(value='x', name='z', initialize=True)
self.assertTrue(np.all(K.eval(K.variable(name='z')) == tmp))
# ====== create by function ====== #
def fn(shape):
return np.full(shape=shape, fill_value=8)
y = K.variable(value=fn,
shape=(12, 18),
dtype='float32',
name='y',
initialize=True)
self.assertTrue(
np.all(K.eval(y) == np.full(shape=(12, 18), fill_value=8)))
# ====== create by initializer ====== #
tmp = K.eval(init_ops.orthogonal_initializer(seed=5218)(shape=(8, 8)))
w = K.variable(value=init_ops.orthogonal_initializer(seed=5218),
shape=(8, 8),
dtype='float32',
name='w',
initialize=True)
self.assertTrue(np.all(K.eval(w) == tmp))
# ====== create by number ====== #
K.variable(value=25,
shape=(8, 8),
dtype='float32',
name='a',
initialize=True)
self.assertTrue(K.eval(K.variable(name='a')).sum() == 25 * 8 * 8)
# ====== create by tensor ====== #
t = tf.constant(value=3,
shape=(12, 8),
dtype='float32',
name='dummy_constant')
K.variable(value=t, name='b', initialize=True)
self.assertTrue(np.all(K.eval(K.variable(name='b')) == K.eval(t)))
# ====== create by Tensor name ====== #
K.variable(value='dummy_constant', name='c', initialize=True)
self.assertTrue(np.all(K.eval(K.variable(name='c')) == K.eval(t)))
# ====== check all variable exist ====== #
all_variables = []
all_variables_name = ['x', 'z', 'y', 'w', 'a', 'b', 'c']
for name in all_variables_name:
v = K.get_all_variables(name=name)
assert len(v) == 1, name
all_variables.append(v[0])
# check no duplicate variables
self.assertTrue(len(set(all_variables)) == len(all_variables_name))
def _apply(self, X, state=None, memory=None):
# time_major: The shape format of the `inputs` and `outputs` Tensors.
# If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`.
# If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`.
# ====== create attention if necessary ====== #
cell = self.cell
if self.bidirectional:
cell_bw = self.cell_bw
# create attention cell
if self.attention:
if not hasattr(self, "_cell_with_attention"):
self._cell_with_attention = self.__attention_creator(
cell, X=X, memory=memory)
cell = self._cell_with_attention
# bidirectional attention
if self.bidirectional:
if not hasattr(self, "_cell_with_attention_bw"):
self._cell_with_attention_bw = self.__attention_creator(
cell_bw, X=X, memory=memory)
cell_bw = self._cell_with_attention_bw
# ====== calling rnn_warpper ====== #
## Bidirectional
if self.bidirectional:
rnn_func = rnn.bidirectional_dynamic_rnn if self.dynamic \
else rnn.static_bidirectional_rnn
state_fw, state_bw = None, None
if isinstance(state, (tuple, list)):
state_fw = state[0]
if len(state) > 1:
state_bw = state[1]
else:
state_fw = state
outputs = rnn_func(cell_fw=cell, cell_bw=cell_bw, inputs=X,
initial_state_fw=state_fw,
initial_state_bw=state_bw,
dtype=X.dtype.base_dtype)
## Unidirectional
else:
rnn_func = rnn.dynamic_rnn if self.dynamic else rnn.static_rnn
outputs = rnn_func(cell, inputs=X, initial_state=state,
dtype=X.dtype.base_dtype)
# ====== initialize cell ====== #
if not self._is_initialized_variables:
# initialize only once, everytime you call this, the values of
# variables changed
K.eval(tf.variables_initializer(self.variables))
self._is_initialized_variables = True
_infer_variable_role(self.variables)
# ====== return ====== #
if self.bidirectional: # concat outputs
outputs = (tf.concat(outputs[0], axis=-1), outputs[1])
if not self.return_states:
return outputs[0]
return outputs
def test_pool_depool(self):
X1 = K.placeholder(shape=(None, 12, 8, 25), name='X1')
X2 = K.placeholder(shape=(None, 12, 8, 25, 18), name='X2')
x1 = np.random.rand(13, 12, 8, 25)
x2 = np.random.rand(13, 12, 8, 25, 18)
prog = Progbar(target=2 * 2 * 2 * 3, print_report=True)
def check_shape(s1, s2):
self.assertEqual(tuple(s1),
tuple(s2),
msg="%s != %s" % (str(s1), str(s2)))
for pool_size in (2, 3):
for strides in (2, 3):
# strides > window_shape not supported due to inconsistency
# between CPU and GPU implementations
if pool_size < strides:
prog.add(1)
continue
for pad in ('valid', 'same'):
for transpose_mode in ('nn', 'pad_margin', 'repeat'):
# ====== print prog ====== #
prog['test'] = "Size:%d,Stride:%d,Pad:%s,T:%s" % \
(pool_size, strides, pad, transpose_mode)
prog.add(1)
# ====== check ops 4D ====== #
down = N.Pool(pool_size=pool_size,
strides=strides,
pad=pad,
mode='max',
transpose_mode=transpose_mode)
up = down.T
y1 = down(X1)
check_shape(
K.eval(y1, {
X1: x1
}).shape[1:],
y1.shape.as_list()[1:])
y2 = up(y1)
check_shape(K.eval(y2, {X1: x1}).shape, x1.shape)
# ====== check ops 5D ====== #
down = N.Pool(pool_size=pool_size,
strides=strides,
pad=pad,
mode='max',
transpose_mode=transpose_mode)
up = down.T
y1 = down(X2)
check_shape(
K.eval(y1, {
X2: x2
}).shape[1:], y1.shape[1:])
y2 = up(y1)
check_shape(K.eval(y2, {X2: x2}).shape, x2.shape)
def test_upsample(self):
X = K.variable(np.arange(1, 24 + 1).reshape(2, 2, 3, 2))
self.assertEqual(K.eval(K.sum(X)), 300.)
self.assertEqual(
K.eval(K.upsample(X, 2, axes=(1, 2), method='nn')).sum(), 1200.)
self.assertEqual(
K.eval(K.upsample(X, 2, axes=(1, 2), method='pad_margin')).sum(),
300.)
self.assertEqual(
K.eval(K.upsample(X, 2, axes=(1, 2), method='repeat')).sum(),
1200.)
def test_linear_algebra_value(self):
np.random.seed(1208)
x = K.variable(np.random.randn(2, 4, 3))
y = K.variable(np.random.rand(1, 2, 3, 5))
z = K.dot(x, y)
self.assertEqual(K.get_shape(z), (2, 4, 1, 2, 5))
self.assertEqual(
repr(np.sum(K.eval(z)))[:8], "-1.0198305134529524"[:8])
np.random.seed(1208)
x = K.variable(np.random.randn(100, 3, 4, 5))
y = K.variable(np.random.rand(100, 12, 5, 6))
z = K.batched_dot(x, y)
self.assertEqual(K.get_shape(z), K.eval(z).shape)
self.assertEqual(repr(K.eval(z).sum())[:7], "1655.44")
def test_confusion_matrix(self):
from sklearn.metrics import confusion_matrix
y1 = np.random.randint(0, 8, size=100)
y2 = np.random.randint(0, 8, size=100)
y_pred = K.variable(y1)
y_true = K.variable(y2)
confusion = K.confusion_matrix(y_pred, y_true)
r1 = K.eval(confusion)
r2 = confusion_matrix(y1, y2)
self.assertEqual(np.sum(r1 - r2), 0.)
def _initialize_param(name, spec, shape):
""" return a ndarray or trainable_variable """
#####################################
# 0. initializing function.
if callable(spec):
spec = spec(shape)
#####################################
# 1. Shared variable, just check the shape.
if K.is_trainable_variable(spec):
spec_shape = K.get_shape(spec)
if not isinstance(spec_shape, tuple):
spec_shape = K.eval(spec_shape)
if shape is None:
shape = spec_shape
elif tuple(shape) != tuple(spec_shape):
raise Exception(
'Given variable has different shape from requirement'
', %s != %s' % (str(spec_shape), str(shape)))
#####################################
# 2. expression, we can only check number of dimension.
elif K.is_variable(spec):
# We cannot check the shape here, Theano expressions (even shared
# variables) do not have a fixed compile-time shape. We can check the
# dimensionality though.
# Note that we cannot assign a name here. We could assign to the
# `name` attribute of the variable, but the user may have already
# named the variable and we don't want to override this.
if shape is not None and K.ndim(spec) != len(shape):
raise Exception(
"parameter with name=%s has %d dimensions, should be "
"%d" % (name, spec.ndim, len(shape)))
#####################################
# 3. numpy ndarray, create shared variable wraper for it.
elif isinstance(spec, np.ndarray):
if shape is not None and spec.shape != shape:
raise RuntimeError(
"parameter with name=%s has shape %s, should be "
"%s" % (name, spec.shape, shape))
#####################################
# 5. Exception.
else:
raise RuntimeError("cannot initialize parameters: 'spec' is not "
"a numpy array, a Theano expression, or a "
"callable")
return spec, shape
def test_randomization(self):
# same rng generate the same random
K.set_rng(12)
a1 = K.eval(K.random_normal(shape=(10, 10), mean=0, std=1))
b1 = K.eval(K.random_uniform(shape=(10, 10), low=-1, high=1))
c1 = K.eval(K.random_binomial(shape=(10, 10), p=0.7))
K.set_rng(12)
a2 = K.eval(K.random_normal(shape=(10, 10), mean=0, std=1))
b2 = K.eval(K.random_uniform(shape=(10, 10), low=-1, high=1))
c2 = K.eval(K.random_binomial(shape=(10, 10), p=0.7))
self.assertTrue(np.array_equal(a1, a2))
self.assertTrue(np.array_equal(b1, b2))
self.assertTrue(np.array_equal(c1, c2))
self.assertTrue(np.abs(np.mean(a1)) < 0.1)
self.assertTrue(np.std(a1) >= 0.8 and np.std(a1) <= 1.2)
self.assertTrue(np.sum(c1) <= 80)
# direct random generation
a = K.eval(K.random_normal(shape=(10, 10)))
b = K.eval(K.random_uniform(shape=(10, 10)))
c = K.eval(K.random_binomial(shape=(10, 10), p=0.1))
self.assertTrue(np.sum(c) <= 20)
def test_computational_graph2(self):
np.random.seed(1208)
X = K.variable(np.zeros((8, 12)), name='X')
Y = K.variable(np.random.rand(12, 8), name='Y')
Z = K.placeholder(shape=(8, 8), name='Z')
a = K.dot(X, Y)
add_roles(a, Auxiliary)
a = a + Z
g1 = K.ComputationGraph(a)
self.assertEqual(len(g1.trainable_variables), 2)
self.assertEqual(len(g1.placeholders), 1)
self.assertEqual(len(g1.updates), 1)
self.assertEqual(len(g1.auxiliary_variables), 1)
f = K.function(Z, [a] + g1.auxiliary_variables)
output = f(np.random.rand(8, 8))
self.assertEqual(repr(np.sum(output[0]))[:5], "32.20")
self.assertEqual(np.sum(output[1]), 0)
self.assertEqual(np.unique(K.eval(X)).tolist(), [12.])
def show_image(x, is_probability=False):
from matplotlib import pyplot as plt
from odin import backend as K
x = anything2image(x)
if x.shape[0] > 32:
x = tf.nn.max_pool(np.reshape(x, (1, x.shape[0], x.shape[1], 1)),
ksize=(1, 4, 4, 1),
strides=(1, 4, 4, 1),
padding="SAME")
x = np.squeeze(K.eval(x))
ax = plt.gca()
plt.imshow(x,
interpolation='nearest',
cmap=plt.cm.Greys_r,
vmin=0. if is_probability else None,
vmax=1. if is_probability else None)
plt.xticks(())
ax.set_xticklabels([])
plt.yticks(())
ax.set_yticklabels([])
ax.set_aspect(aspect='auto')
def test_auto_infer_shape(self):
x = K.variable(np.random.rand(8, 25, 12))
y = K.placeholder((None, 25, 12))
def test_func(func):
self.assertEquals(K.get_shape(func(x, 0)),
K.eval(func(x, 0)).shape)
self.assertEquals(K.get_shape(func(x, -1)),
K.eval(func(x, -1)).shape)
self.assertEquals(K.get_shape(func(x, 1, True)),
K.eval(func(x, 1, True)).shape)
self.assertEquals(K.get_shape(func(x, 0)), K.get_shape(func(y, 0)))
self.assertEquals(K.get_shape(func(x, 0, True)),
K.get_shape(func(y, 0, True)))
if func != K.argmax and func != K.argmin:
self.assertEquals(K.get_shape(func(x, (1, -1))),
K.eval(func(x, (1, -1))).shape)
self.assertEquals(K.get_shape(func(x, (0, 1))),
K.eval(func(x, (0, 1))).shape)
self.assertEquals(K.get_shape(func(x, (0, 1), True)),
K.eval(func(x, (0, 1), True)).shape)
test_func(K.var)
test_func(K.max)
test_func(K.min)
test_func(K.any)
test_func(K.sum)
test_func(K.prod)
test_func(K.mean)
test_func(K.std)
test_func(K.any)
test_func(K.argmax)
test_func(K.argmin)
self.assertEquals(K.get_shape(K.argsort(x)),
K.eval(K.argsort(x)).shape)
def test_func(func):
self.assertEquals(K.get_shape(func(x, 0)),
K.eval(func(x, 0)).shape)
self.assertEquals(K.get_shape(func(x, -1)),
K.eval(func(x, -1)).shape)
self.assertEquals(K.get_shape(func(x, 1, True)),
K.eval(func(x, 1, True)).shape)
self.assertEquals(K.get_shape(func(x, 0)), K.get_shape(func(y, 0)))
self.assertEquals(K.get_shape(func(x, 0, True)),
K.get_shape(func(y, 0, True)))
if func != K.argmax and func != K.argmin:
self.assertEquals(K.get_shape(func(x, (1, -1))),
K.eval(func(x, (1, -1))).shape)
self.assertEquals(K.get_shape(func(x, (0, 1))),
K.eval(func(x, (0, 1))).shape)
self.assertEquals(K.get_shape(func(x, (0, 1), True)),
K.eval(func(x, (0, 1), True)).shape)
def test_ops(self):
x = K.variable(np.random.rand(8, 12))
y = K.variable(np.random.rand(12, 25))
z = K.placeholder((25, 18, 13))
w = K.placeholder((18, 18))
# ====== dot ====== #
t = K.dot(x, y)
self.assertEquals(K.get_shape(t), (8, 25))
self.assertEquals(K.get_shape(t), K.eval(t).shape)
t = K.dot(t, K.dimshuffle(z, (1, 0, 2)))
self.assertEquals(K.get_shape(t), (8, 18, 13))
# ====== transpose ====== #
self.assertEquals(K.get_shape(K.transpose(z)), (13, 18, 25))
self.assertEquals(K.get_shape(K.transpose(t, axes=(2, 0, 1))),
(13, 8, 18))
# ====== eye ====== #
self.assertEquals(K.get_shape(K.eye(5)), K.eval(K.eye(5)).shape)
# ====== diag ====== #
self.assertEquals(K.get_shape(K.diag(w)), (18, ))
# self.assertEquals(K.get_shape(K.diag(x)),
# K.eval(K.diag(y)).shape)
self.assertEquals(K.get_shape(K.square(x)), K.eval(K.square(x)).shape)
self.assertEquals(K.get_shape(K.abs(x)), K.eval(K.abs(x)).shape)
self.assertEquals(K.get_shape(K.sqrt(x)), K.eval(K.sqrt(x)).shape)
self.assertEquals(K.get_shape(K.exp(x)), K.eval(K.exp(x)).shape)
self.assertEquals(K.get_shape(K.log(x)), K.eval(K.log(x)).shape)
self.assertEquals(K.get_shape(K.round(x)), K.eval(K.round(x)).shape)
self.assertEquals(K.get_shape(K.pow(x, 2)), K.eval(K.pow(x, 2)).shape)
self.assertEquals(K.get_shape(K.clip(x, -1, 1)),
K.eval(K.clip(x, -1, 1)).shape)
self.assertEquals(K.get_shape(K.inv(x)), K.eval(K.inv(x)).shape)
def test_cudnn_rnn(self):
if get_ngpu() == 0:
return
print()
batch_size = 2
time_steps = 5
input_dim = 12
hidden_dim = 8
X = K.variable(value=np.random.rand(batch_size, time_steps, input_dim),
dtype='float32',
name='X')
for rnn_mode in ('lstm', 'rnn_relu', 'gru'):
for num_layers in [1, 2]:
for W_init in [
init_ops.glorot_uniform_initializer(seed=1234),
init_ops.random_normal_initializer(seed=1234)
]:
for b_init in [0, 1]:
for bidirectional in (True, False):
for skip_input in (False, ):
print('RNNmode:%s' % rnn_mode,
"#Layers:%d" % num_layers,
'Bidirectional:%s' % bidirectional,
'SkipInput:%s' % skip_input)
weights, biases = K.init_rnn(
input_dim=input_dim,
hidden_dim=hidden_dim,
num_gates=rnn_mode,
num_layers=num_layers,
W_init=W_init,
b_init=b_init,
skip_input=skip_input,
cudnn_vector=False,
is_bidirectional=bidirectional,
name=None)
# ====== check number of params ====== #
params1 = K.params_to_cudnn(weights, biases)
n = params1.shape[0].value
nb_params = cudnn_rnn_ops.cudnn_rnn_opaque_params_size(
rnn_mode=rnn_mode,
num_layers=num_layers,
num_units=hidden_dim,
input_size=input_dim,
input_mode='skip_input'
if skip_input else 'linear_input',
direction='bidirectional'
if bidirectional else 'unidirectional')
nb_params = K.eval(nb_params)
assert n == nb_params
# ====== check cannonical shape match ====== #
kwargs = {
'num_layers':
num_layers,
'num_units':
hidden_dim,
'input_mode':
'skip_input'
if skip_input else 'linear_input',
'direction':
'bidirectional'
if bidirectional else 'unidirectional'
}
if rnn_mode == 'lstm':
rnn = cudnn_rnn.CudnnLSTM(**kwargs)
elif rnn_mode == 'gru':
rnn = cudnn_rnn.CudnnGRU(**kwargs)
if rnn_mode == 'rnn_relu':
rnn = cudnn_rnn.CudnnRNNRelu(**kwargs)
if rnn_mode == 'rnn_tanh':
rnn = cudnn_rnn.CudnnRNNTanh(**kwargs)
rnn.build(input_shape=(None, None, input_dim))
assert len(weights) == len(
rnn.canonical_weight_shapes)
assert len(biases) == len(
rnn.canonical_bias_shapes)
for w, s in zip(weights,
rnn.canonical_weight_shapes):
assert tuple(w.shape.as_list()) == s
# ====== check params conversion ====== #
K.initialize_all_variables()
params2 = cudnn_rnn_ops.cudnn_rnn_canonical_to_opaque_params(
rnn_mode=rnn_mode,
num_layers=num_layers,
num_units=hidden_dim,
input_size=input_dim,
input_mode='skip_input'
if skip_input else 'linear_input',
direction='bidirectional'
if bidirectional else 'unidirectional',
weights=weights,
biases=biases)
assert np.all(
K.eval(params1) == K.eval(params2))
# ====== odin cudnn implementation ====== #
name = 'TEST' + uuid(length=25)
outputs = K.cudnn_rnn(
X=X,
num_units=hidden_dim,
rnn_mode=rnn_mode,
num_layers=num_layers,
parameters=None,
skip_input=skip_input,
is_bidirectional=bidirectional,
dropout=0.1,
name=name)
K.initialize_all_variables()
s0 = K.eval(outputs[0]).sum()
s1 = K.eval(outputs[1]).sum()
all_variables = K.get_all_variables(scope=name)
new_weights = [
i for i in all_variables
if K.role.has_roles(i, roles=K.role.Weight)
]
new_biases = [
i for i in all_variables
if K.role.has_roles(i, roles=K.role.Bias)
]
new_weights, new_biases = K.sort_cudnn_params(
new_weights, new_biases, rnn_mode=rnn_mode)
assert len(weights) == len(weights)
assert len(biases) == len(biases)
for i, j in zip(weights + biases,
new_weights + new_biases):
assert i.name.split(
'/')[-1] == j.name.split('/')[-1]
# ====== CudnnRNN wrapper ====== #
rnn = N.CudnnRNN(
num_units=hidden_dim,
W_init=new_weights,
b_init=new_biases,
rnn_mode=rnn_mode,
num_layers=num_layers,
skip_input=skip_input,
is_bidirectional=bidirectional,
return_states=True,
dropout=0.)
outputs = rnn(X)
K.initialize_all_variables()
y0 = K.eval(outputs[0]).sum()
y1 = K.eval(outputs[1]).sum()
assert y0 == s0
assert y1 == s1
def test_func(func):
y = func(x)
yT = func.T(func(x))
self.assertEquals(K.eval(y).shape, tuple(y.shape.as_list()))
self.assertEquals(K.eval(yT).shape, (25, 8, 12))
self.assertEquals(K.eval(yT).shape, tuple(yT.shape.as_list()))
update_ops = K.optimizers.Adam(lr=0.001).minimize(loss)
K.initialize_all_variables()
# ====== intitalize ====== #
record_train_loss = []
record_valid_loss = []
patience = 3
epoch = 0
# We want the rate to go up but the distortion to go down
while True:
# ====== training ====== #
train_losses = []
prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch)
start_time = timeit.default_timer()
for start, end in batching(batch_size=args.bs, n=X_train.shape[0],
seed=K.get_rng().randint(10e8)):
_ = K.eval(loss, feed_dict={X: X_train[start:end]},
update_after=update_ops)
prog.add(end - start)
train_losses.append(_)
# ====== training log ====== #
print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time))
print("[Training set] Loss: %.4f" % np.mean(train_losses))
# ====== validation set ====== #
code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid})
print("[Valid set] Loss: %.4f" % lo)
# ====== record the history ====== #
record_train_loss.append(np.mean(train_losses))
record_valid_loss.append(lo)
# ====== plotting ====== #
if args.dim > 2:
code_samples = ml.fast_pca(code_samples, n_components=2,
random_state=K.get_rng().randint(10e8))
def test_func(x, y, func):
self.assertEquals(K.get_shape(func(x, y)),
K.eval(func(x, y)).shape)
def test_func(x, func, *args, **kwargs):
self.assertEquals(K.get_shape(func(x, *args, **kwargs)),
K.eval(func(x, *args, **kwargs)).shape)
update_ops = K.optimizers.Adam(lr=0.001).minimize(loss)
K.initialize_all_variables()
# ====== intitalize ====== #
record_train_loss = []
record_valid_loss = []
patience = 3
epoch = 0
# We want the rate to go up but the distortion to go down
while True:
# ====== training ====== #
train_losses = []
prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch)
start_time = timeit.default_timer()
for start, end in batching(batch_size=args.bs, n=X_train.shape[0],
seed=K.get_rng().randint(10e8)):
_ = K.eval(loss, feed_dict={X: X_train[start:end]},
update_after=update_ops)
prog.add(end - start)
train_losses.append(_)
# ====== training log ====== #
print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time))
print("[Training set] Loss: %.4f" % np.mean(train_losses))
# ====== validation set ====== #
code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid})
print("[Valid set] Loss: %.4f" % lo)
# ====== record the history ====== #
record_train_loss.append(np.mean(train_losses))
record_valid_loss.append(lo)
# ====== plotting ====== #
if args.dim > 2:
code_samples = ml.fast_pca(code_samples, n_components=2,
random_state=K.get_rng().randint(10e8))
def test_basic_ops_value(self):
np.random.seed(12082518)
x = K.variable(np.random.randn(8, 8))
y = K.variable(np.random.randn(8, 8))
z = K.variable(np.random.randint(0, 2, size=(8, 8)), dtype=np.bool)
w = K.variable(np.random.randint(0, 2, size=(8, 8)), dtype=np.bool)
self.assertEqual(round(np.sum(K.eval(K.relu(x, alpha=0.12))) * 10000),
276733)
self.assertEqual(round(np.sum(K.eval(K.elu(x, alpha=0.12))) * 10000),
289202)
self.assertEqual(np.sum(K.eval(K.softmax(x))), 8.0)
self.assertEqual(round(np.sum(K.eval(K.softplus(x))) * 10000), 554564)
self.assertEqual(round(np.sum(K.eval(K.softsign(x))) * 100000), 211582)
self.assertEqual(round(np.sum(K.eval(K.sigmoid(x))) * 10000), 330427)
self.assertEqual(round(np.sum(K.eval(K.hard_sigmoid(x))) * 10000),
330836)
self.assertEqual(round(np.sum(K.eval(K.tanh(x))) * 100000), 290165)
self.assertEqual(round(np.sum(K.eval(K.square(x))) * 10000), 744492)
self.assertEqual(round(np.sum(K.eval(K.sqrt(x))) * 10000), 300212)
self.assertEqual(round(np.sum(K.eval(K.abs(x))) * 10000), 559979)
self.assertEqual(np.sum(K.eval(K.sign(x))), 6.0)
self.assertEqual(round(np.sum(K.eval(K.inv(x))) * 1000), 495838)
self.assertEqual(round(np.sum(K.eval(K.exp(x))) * 1000), 122062)
self.assertEqual(round(np.sum(K.eval(K.log(K.abs(x)))) * 10000),
-344491)
self.assertEqual(np.sum(K.eval(K.round(x))), 5.0)
self.assertEqual(round(np.sum(K.eval(K.pow(x, 8))) * 100), 398153)
self.assertEqual(
round(np.sum(K.eval(K.clip(x, -0.12, 0.12))) * 1000000), 620529)
# TODO: pygpu (libgpuarray) still not support diag
# self.assertEqual(round(np.sum(K.eval(K.diag(x))) * 100000), 325289)
self.assertEqual(np.sum(K.eval(K.eye(12, 8))), 8.0)
self.assertEqual(np.sum(K.eval(K.eq(z, w))), 38)
self.assertEqual(np.sum(K.eval(K.neq(z, w))), 26)
self.assertEqual(np.sum(K.eval(K.gt(x, y))), 33)
self.assertEqual(np.sum(K.eval(K.ge(x, y))), 33)
self.assertEqual(np.sum(K.eval(K.lt(x, y))), 31)
self.assertEqual(np.sum(K.eval(K.le(x, y))), 31)
self.assertEqual(round(np.sum(K.eval(K.switch(z, x, y))) * 100000),
139884)
