def __setstate__(self, states):
(
self._seq_ops,
self._input_info, # list: (name, dtype, shape)
self._output_info,
self._path,
self._loss,
self._optimizer,
self._train_args,
self._metric,
self._n_epoch,
self._valid_freq,
self._batch_size,
self._seed,
self._extensions) = states
# ====== create some default values ====== #
self._y_train = None
self._y_pred = None
self._functions = {}
# ====== initialize ====== #
self.set_inputs(*[
K.placeholder(shape=shape, dtype=dtype, name=name)
for name, dtype, shape in self._input_info
])
self.set_outputs(*[
K.placeholder(shape=shape, dtype=dtype, name=name)
for name, dtype, shape in self._output_info
])
def test_simple_rnn(self):
np.random.seed(12082518)
x = np.random.rand(128, 8, 32)
#
X = K.placeholder(shape=(None, 8, 32))
X1 = K.placeholder(shape=(None, 8, 32))
X2 = K.placeholder(shape=(None, 8, 32))
X3 = K.placeholder(shape=(None, 8, 33))
f = N.RNN(32, activation=K.relu, input_mode='skip')
#
y = f(X, mask=K.ones(shape=(128, 8)))
graph = K.ComputationGraph(y)
self.assertEqual(len(graph.inputs), 1)
f1 = K.function([X], y)
x1 = f1(x)
# ====== different placeholder ====== #
y = f(X1)
f2 = K.function([X1], y)
x2 = f1(x)
self.assertEqual(np.sum(x1[0] == x2[0]), np.prod(x1[0].shape))
# ====== pickle load ====== #
f = cPickle.loads(cPickle.dumps(f))
y = f(X2)
f2 = K.function([X2], y)
x3 = f2(x)
self.assertEqual(np.sum(x2[0] == x3[0]), np.prod(x2[0].shape))
# ====== other input shape ====== #
error_happen = False
try:
y = f(X3)
f3 = K.function([X3], y)
x3 = f3(np.random.rand(128, 8, 33))
except (ValueError, Exception):
error_happen = True
self.assertTrue(error_happen)
def test_ops(self):
x = K.variable(np.random.rand(8, 12))
y = K.variable(np.random.rand(12, 25))
z = K.placeholder((25, 18, 13))
w = K.placeholder((18, 18))
# ====== dot ====== #
t = K.dot(x, y)
self.assertEquals(K.get_shape(t), (8, 25))
self.assertEquals(K.get_shape(t), K.eval(t).shape)
t = K.dot(t, K.dimshuffle(z, (1, 0, 2)))
self.assertEquals(K.get_shape(t), (8, 18, 13))
# ====== transpose ====== #
self.assertEquals(K.get_shape(K.transpose(z)), (13, 18, 25))
self.assertEquals(K.get_shape(K.transpose(t, axes=(2, 0, 1))),
(13, 8, 18))
# ====== eye ====== #
self.assertEquals(K.get_shape(K.eye(5)), K.eval(K.eye(5)).shape)
# ====== diag ====== #
self.assertEquals(K.get_shape(K.diag(w)), (18, ))
# self.assertEquals(K.get_shape(K.diag(x)),
# K.eval(K.diag(y)).shape)
self.assertEquals(K.get_shape(K.square(x)), K.eval(K.square(x)).shape)
self.assertEquals(K.get_shape(K.abs(x)), K.eval(K.abs(x)).shape)
self.assertEquals(K.get_shape(K.sqrt(x)), K.eval(K.sqrt(x)).shape)
self.assertEquals(K.get_shape(K.exp(x)), K.eval(K.exp(x)).shape)
self.assertEquals(K.get_shape(K.log(x)), K.eval(K.log(x)).shape)
self.assertEquals(K.get_shape(K.round(x)), K.eval(K.round(x)).shape)
self.assertEquals(K.get_shape(K.pow(x, 2)), K.eval(K.pow(x, 2)).shape)
self.assertEquals(K.get_shape(K.clip(x, -1, 1)),
K.eval(K.clip(x, -1, 1)).shape)
self.assertEquals(K.get_shape(K.inv(x)), K.eval(K.inv(x)).shape)
def _initialize(self, X, y=None):
with tf.name_scope(self.name):
# ====== input_shape ====== #
if self._input_shape is None:
self._input_shape = X.shape
elif self.input_shape[1:] != X.shape[1:]:
raise ValueError(
"Initialized with input shape: %s, given tensor with shape: %s"
% (self.input_shape, X.shape))
# ====== output_shape ====== #
if self._output_shape is None:
self._output_shape = y.shape
elif self.output_shape[1:] != y.shape[1:]:
raise ValueError(
"Initialized with output shape: %s, given tensor with shape: %s"
% (self.output_shape, y.shape))
# ====== placeholder ====== #
self._X = K.placeholder(shape=self.input_shape,
dtype=self.dtype,
name='input')
self._y = K.placeholder(shape=self.output_shape,
dtype=self.dtype,
name='output')
# ====== run the network ====== #
y_pred_logits = self.network.apply(self._X)
nb_classes = y_pred_logits.shape.as_list()[-1]
if len(self._output_shape) == 1:
y_true = tf.one_hot(indices=tf.cast(self._y, 'int32'),
depth=nb_classes)
elif self._output_shape[-1] != nb_classes:
raise ValueError(
"Given %d classes, but output from network has %s classes"
% (self._output_shape[-1], nb_classes))
self._nb_classes = nb_classes
# ====== sigmoid or softmax ====== #
if nb_classes == 2:
fn_activation = tf.nn.sigmoid
fn_loss = tf.losses.sigmoid_cross_entropy
fn_acc = K.metrics.binary_accuracy
else:
fn_activation = tf.nn.softmax
fn_loss = tf.losses.softmax_cross_entropy
fn_acc = K.metrics.categorical_accuracy
y_pred_proba = fn_activation(y_pred_logits)
# ====== class weight ====== #
class_weights = np.ones(shape=(nb_classes,), dtype=self.dtype) if self._class_weights is None\
else as_tuple(self._class_weights, N=self.nb_classes, t=float)
class_weights = tf.constant(value=class_weights,
dtype=self.dtype,
name="class_weights")
weights = tf.gather(
class_weights,
tf.cast(self._y, 'int32')
if self.nb_classes == 2 else tf.argmax(self._y, axis=-1))
# ====== objectives ====== #
cost_train = fn_loss(y_true, logits=y_pred_logits, weights=weights)
exit()
def test_conv_deconv_transpose(self):
def feval(X, y):
f = K.function(X, y)
shape = (np.random.randint(8, 18), ) + tuple(X.shape.as_list()[1:])
x = np.random.rand(*shape)
return f(x)
prog = Progbar(target=2 * 3 * 3 * 2 * 2, print_report=True)
for X in (K.placeholder(shape=(None, 13, 12, 25)),
K.placeholder(shape=(None, 13, 12, 8, 25))):
for strides in (1, 2, 3):
for filter_size in (3, 4, 5):
for num_filters in (8, 25):
for pad in ("same", "valid"):
for dilation in (1, ):
# ====== progress ====== #
prog['test'] = "#Dim:%d;Stride:%d;Filter:%d;Channel:%d;Pad:%s" % \
(X.shape.ndims, strides, filter_size, num_filters, pad)
prog.add(1)
# ====== test Conv ====== #
f = N.Conv(num_filters=num_filters,
filter_size=filter_size,
pad=pad,
strides=strides,
activation=tf.nn.relu,
dilation=dilation)
fT = f.T
y = f(X)
self.assertEqual(
feval(X, y).shape[1:],
tuple(y.shape.as_list()[1:]))
yT = fT(y)
self.assertEqual(
feval(X, yT).shape[1:],
tuple(yT.shape.as_list()[1:]))
self.assertEqual(X.shape.as_list(),
yT.shape.as_list())
# ====== test Transpose ====== #
f = N.TransposeConv(num_filters=num_filters,
filter_size=filter_size,
pad=pad,
strides=strides,
activation=K.relu,
dilation=dilation)
fT = f.T
y = f(X)
self.assertEqual(
feval(X, y).shape[1:],
tuple(y.shape.as_list()[1:]))
yT = fT(y)
self.assertEqual(
feval(X, yT).shape[1:],
tuple(yT.shape.as_list()[1:]))
self.assertEqual(X.shape.as_list(),
yT.shape.as_list())
def test_pool_depool(self):
X1 = K.placeholder(shape=(None, 12, 8, 25), name='X1')
X2 = K.placeholder(shape=(None, 12, 8, 25, 18), name='X2')
x1 = np.random.rand(13, 12, 8, 25)
x2 = np.random.rand(13, 12, 8, 25, 18)
prog = Progbar(target=2 * 2 * 2 * 3, print_report=True)
def check_shape(s1, s2):
self.assertEqual(tuple(s1),
tuple(s2),
msg="%s != %s" % (str(s1), str(s2)))
for pool_size in (2, 3):
for strides in (2, 3):
# strides > window_shape not supported due to inconsistency
# between CPU and GPU implementations
if pool_size < strides:
prog.add(1)
continue
for pad in ('valid', 'same'):
for transpose_mode in ('nn', 'pad_margin', 'repeat'):
# ====== print prog ====== #
prog['test'] = "Size:%d,Stride:%d,Pad:%s,T:%s" % \
(pool_size, strides, pad, transpose_mode)
prog.add(1)
# ====== check ops 4D ====== #
down = N.Pool(pool_size=pool_size,
strides=strides,
pad=pad,
mode='max',
transpose_mode=transpose_mode)
up = down.T
y1 = down(X1)
check_shape(
K.eval(y1, {
X1: x1
}).shape[1:],
y1.shape.as_list()[1:])
y2 = up(y1)
check_shape(K.eval(y2, {X1: x1}).shape, x1.shape)
# ====== check ops 5D ====== #
down = N.Pool(pool_size=pool_size,
strides=strides,
pad=pad,
mode='max',
transpose_mode=transpose_mode)
up = down.T
y1 = down(X2)
check_shape(
K.eval(y1, {
X2: x2
}).shape[1:], y1.shape[1:])
y2 = up(y1)
check_shape(K.eval(y2, {X2: x2}).shape, x2.shape)
def test_batch_norm(self):
K.set_training(True)
x = K.placeholder((None, 8, 12))
y = N.BatchNorm()(x)
f = K.function(x, y)
z = f(np.random.rand(25, 8, 12))
self.assertEquals(z.shape, (25, 8, 12))
# ====== Not training ====== #
K.set_training(False)
x = K.placeholder((None, 8, 12))
y = N.BatchNorm()(x)
f = K.function(x, y)
z = f(np.random.rand(25, 8, 12))
self.assertEquals(z.shape, (25, 8, 12))
def _initialize(self, X, y=None):
with tf.name_scope(self.name):
# ====== input_shape ====== #
if self._input_shape is None:
self._input_shape = X.shape
elif self.input_shape[1:] != X.shape[1:]:
raise ValueError("Initialized with input shape: %s, given tensor with shape: %s"
% (self.input_shape, X.shape))
# ====== output_shape ====== #
if self._output_shape is None:
self._output_shape = y.shape
elif self.output_shape[1:] != y.shape[1:]:
raise ValueError("Initialized with output shape: %s, given tensor with shape: %s"
% (self.output_shape, y.shape))
# ====== placeholder ====== #
self._X = K.placeholder(shape=self.input_shape, dtype=self.dtype, name='input')
self._y = K.placeholder(shape=self.output_shape, dtype=self.dtype, name='output')
# ====== run the network ====== #
y_pred_logits = self.network.apply(self._X)
nb_classes = y_pred_logits.shape.as_list()[-1]
if len(self._output_shape) == 1:
y_true = tf.one_hot(indices=tf.cast(self._y, 'int32'),
depth=nb_classes)
elif self._output_shape[-1] != nb_classes:
raise ValueError("Given %d classes, but output from network has %s classes" %
(self._output_shape[-1], nb_classes))
self._nb_classes = nb_classes
# ====== sigmoid or softmax ====== #
if nb_classes == 2:
fn_activation = tf.nn.sigmoid
fn_loss = tf.losses.sigmoid_cross_entropy
fn_acc = K.metrics.binary_accuracy
else:
fn_activation = tf.nn.softmax
fn_loss = tf.losses.softmax_cross_entropy
fn_acc = K.metrics.categorical_accuracy
y_pred_proba = fn_activation(y_pred_logits)
# ====== class weight ====== #
class_weights = np.ones(shape=(nb_classes,), dtype=self.dtype) if self._class_weights is None\
else as_tuple(self._class_weights, N=self.nb_classes, t=float)
class_weights = tf.constant(value=class_weights, dtype=self.dtype,
name="class_weights")
weights = tf.gather(class_weights,
tf.cast(self._y, 'int32') if self.nb_classes == 2 else
tf.argmax(self._y, axis=-1))
# ====== objectives ====== #
cost_train = fn_loss(y_true, logits=y_pred_logits, weights=weights)
exit()
def test_computational_graph3(self):
# validate the number of updates found by ComputationGraph
X = K.placeholder(shape=(None, 28, 28, 3))
f = N.Sequence([
N.Conv(32, 3, pad='same', activation=K.linear),
N.BatchNorm(activation=K.relu),
N.Flatten(outdim=2),
N.Dense(16),
N.BatchNorm(),
N.Dense(10)
])
K.set_training(True)
y_train = f(X)
K.set_training(False)
y_score = f(X)
self.assertTrue(
K.get_shape(y_train) == K.get_shape(y_score)
and K.get_shape(y_score) == (None, 10))
cc_train = K.ComputationGraph(y_train)
cc_score = K.ComputationGraph(y_score)
self.assertTrue(len(cc_score.updates) == 0)
self.assertTrue(len(cc_train.updates) == 4)
# create real function
fn_train = K.function(X, y_train)
fn_score = K.function(X, y_score)
shape1 = fn_train(np.random.rand(12, 28, 28, 3)).shape
shape2 = fn_score(np.random.rand(12, 28, 28, 3)).shape
self.assertTrue(shape1 == shape2 and shape1 == (12, 10))
def test_seq(self):
X = K.placeholder((None, 28, 28, 1))
f = N.Sequence([
N.Conv(8, (3, 3), strides=1, pad='same'),
N.Dimshuffle(pattern=(0, 3, 1, 2)),
N.Flatten(outdim=2),
N.Noise(level=0.3, noise_dims=None, noise_type='gaussian'),
N.Dense(128, activation=tf.nn.relu),
N.Dropout(level=0.3, noise_dims=None),
N.Dense(10, activation=tf.nn.softmax)
])
y = f(X)
yT = f.T(y)
f1 = K.function(X, y, defaults={K.is_training(): True})
f2 = K.function(X, yT, defaults={K.is_training(): False})
f = cPickle.loads(cPickle.dumps(f))
y = f(X)
yT = f.T(y)
f3 = K.function(X, y, defaults={K.is_training(): True})
f4 = K.function(X, yT, defaults={K.is_training(): False})
x = np.random.rand(12, 28, 28, 1)
self.assertEquals(f1(x).shape, (2688, 10))
self.assertEquals(f3(x).shape, (2688, 10))
self.assertEqual(np.round(f1(x).sum(), 4), np.round(f3(x).sum(), 4))
self.assertEquals(y.shape.as_list(), (None, 10))
self.assertEquals(f2(x).shape, (12, 28, 28, 1))
self.assertEquals(f4(x).shape, (12, 28, 28, 1))
self.assertEqual(str(f2(x).sum())[:4], str(f4(x).sum())[:4])
self.assertEquals(yT.shape.as_list(), (None, 28, 28, 1))
def test_dilatedConv(self):
x = K.placeholder((None, 28, 28, 3))
f1 = N.Conv(16, (3, 3), dilation=(2, 2))
y = f1(x)
f = K.function(x, y)
z = f(np.random.rand(12, 28, 28, 3))
self.assertEquals(z.shape, (12, 24, 24, 16))
self.assertEquals(y.shape.as_list(), [None, 24, 24, 16])
def test_conv3D(self):
x = K.placeholder((None, 28, 28, 28, 3))
f1 = N.Conv(16, (3, 3, 3), strides=1, pad='valid')
y = f1(x)
f = K.function(x, y)
z = f(np.random.rand(12, 28, 28, 28, 3))
self.assertEquals(z.shape, (12, 26, 26, 26, 16))
self.assertEquals(y.shape.as_list(), [None, 26, 26, 26, 16])
def test_flatten(self):
x = K.placeholder(shape=(None, 8, 12, 25, 18))
for i in range(1, 5):
y = K.flatten(x, outdim=i)
f = K.function(x, y)
shape1 = K.get_shape(y)
shape2 = f(np.random.rand(16, 8, 12, 25, 18)).shape
self.assertEqual(len(shape1), len(shape2))
self.assertTrue(
all(i == j for i, j in zip(shape1, shape2) if i is not None))
def _auto_create_inputs(self, X):
if len(self._inputs) > 0:
return
if not isinstance(X, (tuple, list)):
X = (X, )
X = [
K.placeholder(shape=(None, ) + i.shape[1:],
dtype=i.dtype,
name='input%d' % _) for _, i in enumerate(X)
]
self.set_inputs(*X)
def test_helper_ops_variables(self):
X = K.placeholder(shape=(10, 20))
f = N.Sequence([
N.Dense(12),
N.Dense(8),
N.BatchNorm(),
N.Dense(25, W_init=tf.zeros(shape=(8, 25)))
])
y = f(X)
self.assertEqual(y.shape.as_list(), [10, 25])
self.assertEqual(len(f.variables), 10)
self.assertEqual(len(f.parameters), 7)
self.assertEqual(len(f.trainable_variables), 9)
def test_slice_ops(self):
X = K.placeholder(shape=(None, 28, 28, 28, 3))
f = N.Sequence([
N.Conv(32, 3, pad='same', activation=K.linear),
N.BatchNorm(activation=tf.nn.relu),
N.Flatten(outdim=4)[:, 8:12, 18:25, 13:],
])
y = f(X)
fn = K.function(X, y)
self.assertTrue(
fn(np.random.rand(12, 28, 28, 28, 3)).shape[1:] == tuple(
y.shape.as_list()[1:]))
self.assertEqual(y.shape.as_list()[1:], [4, 7, 883])
def test_cudnn_rnn_backend(self):
if get_device() == 'cpu':
return
print()
np.random.seed(1208)
batch_size = 25
hidden_size = 12
X_linear = K.placeholder(shape=(None, 8, 32), name='X_linear')
X_skip = K.placeholder(shape=(None, 8, 12), name='X_skip')
for direction_mode in ['bidirectional', 'unidirectional']:
for nb_layers in [1, 2, 3]:
for rnn_mode in ['gru', 'lstm', 'rnn_tanh']:
for input_mode in ['linear', 'skip']:
if input_mode == 'linear':
X = X_linear
x = np.random.rand(batch_size, 8, 32)
else:
X = X_skip
x = np.random.rand(batch_size, 8, 12)
start = timeit.default_timer()
y = K.rnn_dnn(X,
hidden_size=hidden_size,
rnn_mode=rnn_mode,
input_mode=input_mode,
num_layers=nb_layers,
direction_mode=direction_mode)
# perform function
f = K.function(X, y)
output = f(x)
benchmark = timeit.default_timer() - start
self.assertEqual([list(i.shape) for i in output], [[
batch_size if j is None else j
for j in K.get_shape(i)
] for i in y])
print(
"*PASSED* [Layers]%s [Mode]%-8s [Input]%-6s [Direction]%s [Benchmark]%.4f"
% (nb_layers, rnn_mode, input_mode, direction_mode,
benchmark))
def inputs(self):
""" Create list of placeholder based on footprint(shape) from previous
inputs of this Operator
"""
if self._configuration is None:
raise Exception("This operators haven't initialized.")
if id(self) in _cached_placeholder:
return _cached_placeholder[id(self)]
inputs = [
K.placeholder(shape=j, name='%s_input%d' % (self.name, i))
for i, j in enumerate(self._footprint)
]
_cached_placeholder[id(self)] = inputs
return inputs
def test_dense(self):
x = K.placeholder((None, 10))
f1 = N.Dense(20)
f2 = N.Dense(30)
y = f2(f1(x))
y = f1.T(f2.T(y))
f = K.function(x, y)
x = f(np.random.rand(12, 10))
self.assertEquals(x.shape, (12, 10))
self.assertEquals(y.shape.as_list(), [None, 10])
def test_noise(self):
x = K.placeholder((2, 3))
f1 = N.Noise(level=0.5, noise_dims=0, noise_type='gaussian')
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((2, 3)))
z = z.tolist()
self.assertTrue(all(i == z[0] for i in z))
f1 = N.Noise(level=0.5, noise_dims=1, noise_type='gaussian')
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((2, 3)))
z = z.T.tolist()
self.assertTrue(all(i == z[0] for i in z))
def test_dropout(self):
x = K.placeholder((4, 6))
f1 = N.Dropout(level=0.5, noise_dims=0, rescale=True)
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((4, 6)))
z = z.tolist()
self.assertTrue(all(i == z[0] for i in z))
f1 = N.Dropout(level=0.5, noise_dims=1, rescale=True)
y = f1(x)
f = K.function(x, y, defaults={K.is_training(): True})
z = f(np.ones((4, 6)))
z = z.T.tolist()
self.assertTrue(all(i == z[0] for i in z))
def test_conv2D(self):
x = K.placeholder((None, 28, 28, 3))
f1 = N.Conv(16, (3, 3), strides=(2, 2), pad='same')
y = f1(x)
f = K.function(x, y)
z = f(np.random.rand(12, 28, 28, 3))
self.assertEquals(z.shape, (12, 14, 14, 16))
self.assertEquals(y.shape.as_list(), [None, 14, 14, 16])
# ====== transpose convolution ====== #
y = f1.T(y)
f = K.function(x, y)
z = f(np.random.rand(12, 28, 28, 3))
self.assertEquals(z.shape, (12, 28, 28, 3))
self.assertEquals(y.shape.as_list(), [None, 28, 28, 3])
def test_computational_graph1(self):
X = K.placeholder(shape=(None, 32), name='input')
z = K.variable(np.random.rand(10, 10), name='z')
f = N.Sequence(
[N.Dense(16, activation=K.relu),
N.Dense(8, activation=K.softmax)])
y = f(X)
add_auxiliary_variable(y, K.constant(10, name='aux_const'))
tmp = K.ComputationGraph(y)
self.assertEqual(len(tmp.placeholders), 1)
self.assertEqual(len(tmp.trainable_variables), 4)
self.assertEqual(len(tmp.parameters), 4)
self.assertEqual(len(tmp.dict_of_placeholders), 1)
self.assertEqual(len(tmp.auxiliary_variables), 1)
tmp.intermediary_variables # no idea how to test this
self.assertEqual(len(tmp.updates), 1)
self.assertEqual(K.ComputationGraph(y), tmp)
def test_load_save3(self):
X = K.placeholder(shape=(None, 28, 28))
ops = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(8, (3, 3), strides=(1, 1), pad='same', activation=K.relu),
K.pool2d,
N.Flatten(outdim=2),
N.Dense(64, activation=K.relu),
N.Dense(10, activation=K.softmax)
])
y = ops(X)
f1 = K.function(X, y)
ops_ = cPickle.loads(cPickle.dumps(ops, protocol=cPickle.HIGHEST_PROTOCOL))
y_ = ops_(X)
f2 = K.function(X, y_)
x = np.random.rand(32, 28, 28)
self.assertEqual(np.sum(f1(x) - f2(x)), 0.)
def set_inputs(self, *inputs):
self._input_info = []
self._inputs = []
for i in inputs:
if not K.is_placeholder(i):
raise ValueError('Only accept input which is placeholder.')
name, dtype, shape = i.name, i.dtype, K.get_shape(i)
self._input_info.append([name, dtype, shape])
self._inputs.append(i)
# ====== Try to check if the inputs match the Ops ====== #
try:
# call this to initialize the parameters and get
# estimated output shape (we assume training and deploying
# mode get the same shape).
for i in self._inputs:
add_role(i, TRAINING)
self._y_train = self._seq_ops(*self._inputs)
for i in self._inputs:
add_role(i, DEPLOYING)
self._y_pred = self._seq_ops(*self._inputs)
# create default output
if len(self._output_info) == 0:
shape = K.get_shape(self._y_train)
self._outputs = [
K.placeholder(shape=shape,
dtype=self._y_train.dtype,
name='output1')
]
self._output_info = [('output1', self._y_train.dtype, shape)]
# reset all functions
for i, j in self._functions.items():
del self._functions[i]
del j
self._functions = {}
except Exception, e:
warnings.warn('Inputs do not match the Ops requirements, '
'error: ' + str(e))
self._input_info = []
self._inputs = []
def test_load_save2(self):
K.set_training(True)
X = K.placeholder((None, 1, 28, 28))
f = N.Dense(128, activation=K.relu)
y = f(X)
yT = f.T(y)
f1 = K.function(X, y)
f2 = K.function(X, yT)
f = cPickle.loads(cPickle.dumps(f))
y = f(X)
yT = f.T(y)
f3 = K.function(X, y)
f4 = K.function(X, yT)
x = np.random.rand(12, 1, 28, 28)
self.assertEqual(f1(x).sum(), f3(x).sum())
self.assertEqual(f2(x).sum(), f4(x).sum())
def test_load_save2(self):
K.set_training(True)
X = K.placeholder((None, 1, 28, 28))
f = N.Dense(128, activation=K.relu)
y = f(X)
yT = f.T(y)
f1 = K.function(X, y)
f2 = K.function(X, yT)
f = cPickle.loads(cPickle.dumps(f))
y = f(X)
yT = f.T(y)
f3 = K.function(X, y)
f4 = K.function(X, yT)
x = np.random.rand(12, 1, 28, 28)
self.assertEqual(f1(x).sum(), f3(x).sum())
self.assertEqual(f2(x).sum(), f4(x).sum())
def test_load_save3(self):
X = K.placeholder(shape=(None, 28, 28))
ops = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(8, (3, 3), strides=(1, 1), pad='same', activation=K.relu),
K.pool2d,
N.Flatten(outdim=2),
N.Dense(64, activation=K.relu),
N.Dense(10, activation=K.softmax)
])
y = ops(X)
f1 = K.function(X, y)
ops_ = cPickle.loads(
cPickle.dumps(ops, protocol=cPickle.HIGHEST_PROTOCOL))
y_ = ops_(X)
f2 = K.function(X, y_)
x = np.random.rand(32, 28, 28)
self.assertEqual(np.sum(f1(x) - f2(x)), 0.)
def test_computational_graph2(self):
np.random.seed(1208)
X = K.variable(np.zeros((8, 12)), name='X')
Y = K.variable(np.random.rand(12, 8), name='Y')
Z = K.placeholder(shape=(8, 8), name='Z')
a = K.dot(X, Y)
add_roles(a, Auxiliary)
a = a + Z
g1 = K.ComputationGraph(a)
self.assertEqual(len(g1.trainable_variables), 2)
self.assertEqual(len(g1.placeholders), 1)
self.assertEqual(len(g1.updates), 1)
self.assertEqual(len(g1.auxiliary_variables), 1)
f = K.function(Z, [a] + g1.auxiliary_variables)
output = f(np.random.rand(8, 8))
self.assertEqual(repr(np.sum(output[0]))[:5], "32.20")
self.assertEqual(np.sum(output[1]), 0)
self.assertEqual(np.unique(K.eval(X)).tolist(), [12.])
def test_auto_infer_shape(self):
x = K.variable(np.random.rand(8, 25, 12))
y = K.placeholder((None, 25, 12))
def test_func(func):
self.assertEquals(K.get_shape(func(x, 0)),
K.eval(func(x, 0)).shape)
self.assertEquals(K.get_shape(func(x, -1)),
K.eval(func(x, -1)).shape)
self.assertEquals(K.get_shape(func(x, 1, True)),
K.eval(func(x, 1, True)).shape)
self.assertEquals(K.get_shape(func(x, 0)), K.get_shape(func(y, 0)))
self.assertEquals(K.get_shape(func(x, 0, True)),
K.get_shape(func(y, 0, True)))
if func != K.argmax and func != K.argmin:
self.assertEquals(K.get_shape(func(x, (1, -1))),
K.eval(func(x, (1, -1))).shape)
self.assertEquals(K.get_shape(func(x, (0, 1))),
K.eval(func(x, (0, 1))).shape)
self.assertEquals(K.get_shape(func(x, (0, 1), True)),
K.eval(func(x, (0, 1), True)).shape)
test_func(K.var)
test_func(K.max)
test_func(K.min)
test_func(K.any)
test_func(K.sum)
test_func(K.prod)
test_func(K.mean)
test_func(K.std)
test_func(K.any)
test_func(K.argmax)
test_func(K.argmin)
self.assertEquals(K.get_shape(K.argsort(x)),
K.eval(K.argsort(x)).shape)
def test_load_save1(self):
K.set_training(True)
X = K.placeholder((None, 1, 28, 28))
f = N.Dense(128, activation=K.relu)
y = f(X)
W, b = [K.get_value(p).sum() for p in K.ComputationGraph(y).parameters]
num_units = f.num_units
W_init = f.W_init
b_init = f.b_init
activation = f.activation
f = cPickle.loads(cPickle.dumps(f))
W1, b1 = [K.get_value(p).sum() for p in f.parameters]
num_units1 = f.num_units
W_init1 = f.W_init
b_init1 = f.b_init
activation1 = f.activation
self.assertEqual(W1, W)
self.assertEqual(b1, b)
self.assertEqual(num_units1, num_units)
self.assertEqual(W_init1.__name__, W_init.__name__)
self.assertEqual(b_init.__name__, b_init1.__name__)
self.assertEqual(activation1, activation)
def test_load_save1(self):
K.set_training(True)
X = K.placeholder((None, 1, 28, 28))
f = N.Dense(128, activation=K.relu)
y = f(X)
W, b = [K.get_value(p).sum() for p in K.ComputationGraph(y).parameters]
num_units = f.num_units
W_init = f.W_init
b_init = f.b_init
activation = f.activation
f = cPickle.loads(cPickle.dumps(f))
W1, b1 = [K.get_value(p).sum() for p in f.parameters]
num_units1 = f.num_units
W_init1 = f.W_init
b_init1 = f.b_init
activation1 = f.activation
self.assertEqual(W1, W)
self.assertEqual(b1, b)
self.assertEqual(num_units1, num_units)
self.assertEqual(W_init1.__name__, W_init.__name__)
self.assertEqual(b_init.__name__, b_init1.__name__)
self.assertEqual(activation1, activation)
NB_EPOCH = 10
LEARNING_RATE = 0.001
# ===========================================================================
# Load dataset
# ===========================================================================
ds = F.CIFAR10.get_dataset()
nb_labels = 10
print(ds)
X_train = ds['X_train'][:].astype('float32') / 255.
y_train = one_hot(ds['y_train'][:], nb_classes=nb_labels)
X_test = ds['X_test'][:].astype('float32') / 255.
y_test = one_hot(ds['y_test'][:], nb_classes=nb_labels)
# ===========================================================================
# Create network
# ===========================================================================
inputs = [K.placeholder(shape=(None,) + X_train.shape[1:], name='X', dtype='float32'),
K.placeholder(shape=(None, nb_labels), name='y', dtype='float32')]
print("Inputs:", inputs)
model = N.Lambda.search(MODEL_NAME, prefix='models_cifar')
outputs = model(*inputs)
# ====== create losses ====== #
ce = tf.losses.softmax_cross_entropy(inputs[-1], outputs['logit'])
acc = K.metrics.categorical_accuracy(outputs['prob'], inputs[-1])
cm = K.metrics.confusion_matrix(y_pred=outputs['prob'],
y_true=inputs[-1],
labels=nb_labels)
# ====== create optimizer ====== #
optz = K.optimizers.Adam(lr=LEARNING_RATE)
parameters = model.parameters
print("#Parameters:", len(parameters))
updates = optz(ce, parameters)
X_valid, y_valid = X_train[40000:], y_train[40000:]
X_train, y_train = X_train[:40000], y_train[:40000]
# normalize value to [0, 1]
X_train = X_train / 255.
X_valid = X_valid / 255.
X_test = X_test / 255.
print(ds)
# ====== others ====== #
X_samples, y_samples = X_train[:25], y_train[:25]
input_shape = ds['X_train'].shape
input_ndim = len(input_shape)
print("Train shape:", ctext(X_train.shape, 'cyan'))
print("Valid shape:", ctext(X_valid.shape, 'cyan'))
print("Test shape:", ctext(X_test.shape, 'cyan'))
# ====== create basic tensor ====== #
X = K.placeholder(shape=(None,) + input_shape[1:], name='X_input')
y = K.placeholder(shape=(None,), name='y_input')
# ===========================================================================
# Create the network
# ===========================================================================
LATENT_DROPOUT = 0.3
if args.cnn:
with N.args_scope(([N.Conv, N.Dense], dict(b_init=None, activation=K.linear)),
(N.BatchNorm, dict(activation=tf.nn.elu)),
(N.Pool, dict(mode='max', pool_size=2))):
f_encoder = N.Sequence([
N.Dropout(level=0.5),
N.Dimshuffle((0, 2, 3, 1)) if is_cifar10 else N.Dimshuffle((0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=3, pad='valid'),
N.Pool(),
def test_odin_vs_lasagne(self):
X1 = K.placeholder(shape=(None, 28, 28))
X2 = K.placeholder(shape=(None, 784))
def lasagne_net1():
"FNN"
i = lasagne.layers.InputLayer(shape=(None, 784))
i.input_var = X2
i = lasagne.layers.DenseLayer(i, num_units=32, W=random(784, 32), b=zeros(32),
nonlinearity=lasagne.nonlinearities.rectify)
i = lasagne.layers.DenseLayer(i, num_units=16, W=random(32, 16), b=zeros(16),
nonlinearity=lasagne.nonlinearities.softmax)
return X2, lasagne.layers.get_output(i)
def odin_net1():
"FNN"
f = N.Sequence([
N.Dense(32, W_init=random(784, 32), b_init=zeros(32),
activation=K.relu),
N.Dense(16, W_init=random(32, 16), b_init=zeros(16),
activation=K.softmax)
])
return X2, f(X2)
def lasagne_net2():
"CNN"
i = lasagne.layers.InputLayer(shape=(None, 28, 28))
i.input_var = X1
i = lasagne.layers.DimshuffleLayer(i, (0, 'x', 1, 2))
i = lasagne.layers.Conv2DLayer(i, 12, (3, 3), stride=(1, 1), pad='same',
untie_biases=False,
W=random(12, 1, 3, 3),
nonlinearity=lasagne.nonlinearities.rectify)
i = lasagne.layers.Pool2DLayer(i, pool_size=(2, 2), stride=None, mode='max',
ignore_border=True)
i = lasagne.layers.Conv2DLayer(i, 16, (3, 3), stride=(1, 1), pad='same',
untie_biases=False,
W=random(16, 12, 3, 3),
nonlinearity=lasagne.nonlinearities.sigmoid)
return X1, lasagne.layers.get_output(i)
def odin_net2():
"CNN"
f = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')),
N.Conv(12, (3, 3), strides=(1, 1), pad='same',
untie_biases=False,
W_init=random(3, 3, 1, 12),
activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None, mode='max'),
N.Conv(16, (3, 3), strides=(1, 1), pad='same',
untie_biases=False,
W_init=random(3, 3, 12, 16),
activation=K.sigmoid),
N.Dimshuffle((0, 3, 1, 2))
])
return X1, f(X1)
def lasagne_net3():
"RNN"
i = lasagne.layers.InputLayer(shape=(None, 28, 28))
i.input_var = X1
W = [random(28, 32), random(32, 32), random(32), random_bin(12, 28)]
i = lasagne.layers.RecurrentLayer(i, num_units=32,
W_in_to_hid=W[0],
W_hid_to_hid=W[1],
b=W[2],
nonlinearity=lasagne.nonlinearities.rectify,
hid_init=zeros(1, 32),
backwards=False,
learn_init=False,
gradient_steps=-1,
grad_clipping=0,
unroll_scan=False,
precompute_input=True,
mask_input=None,
only_return_final=False)
return X1, lasagne.layers.get_output(i)
def odin_net3():
"RNN"
W = [random(28, 32), random(32, 32), random(32), random_bin(12, 28)]
f = N.Sequence([
N.Dense(num_units=32, W_init=W[0], b_init=W[2],
activation=K.linear),
N.RNN(num_units=32, activation=K.relu,
W_init=W[1])
])
return X1, f(X1, hid_init=zeros(1, 32))
func_list = [
(lasagne_net1, odin_net1),
# (lasagne_net2, odin_net2),
(lasagne_net3, odin_net3)
]
print()
for i, j in func_list:
print('Test:', i.__name__, j.__name__)
seed = np.random.randint(10e8)
# ====== call the function ====== #
np.random.seed(seed)
i = i()
np.random.seed(seed)
j = j()
# ====== create theano function ====== #
f1 = K.function(i[0], i[1])
f2 = K.function(j[0], j[1])
shape = K.get_shape(i[0])
# ====== get the output ====== #
x = np.random.rand(*[12 if s is None else s for s in shape])
y1 = f1(x)
y2 = f2(x)
self.assertEqual(y1.shape, y2.shape)
self.assertAlmostEqual(np.sum(np.abs(y1 - y2)), 0.)
parallel_iterations=32, back_prop=True,
swap_memory=False, infer_shape=True,
name=name)
# consistent return as theano
if nb_outputs == 1:
outputs = outputs[0]
return outputs
# ====== simulate data ====== #
def doit(_, x, y, z):
z += K.sum(x + y) + K.sum(K.pow(_, 2))
return z
sequences = [
K.placeholder(shape=(600, None)),
K.variable(np.arange(0, 1200).reshape(-1, 2)),
K.variable(np.arange(1200, 2400).reshape(-1, 2))
]
outputs_info = K.zeros(shape=(1200,))
X = np.random.rand(600, 3000)
# ====== tf.scan ====== #
y = Scan2(doit,
sequences=sequences,
outputs_info=outputs_info,
n_steps=None,
backwards=True,
name=None)
print('Scan:')
y_train = y[:int(0.8 * n)]
X_valid = X[int(0.8 * n):]
y_valid = y[int(0.8 * n):]
print('X:', X.shape, 'y:', y.shape)
print('X_train:', X_train.shape, 'y_train:', y_train.shape)
print('X_valid:', X_valid.shape, 'y_valid:', y_valid.shape)
E = tk.embed(embedding)
# these numbers must be the same for all time
print('Tokenizer:', np.sum(E), np.sum(X_train), np.sum(y_train),
np.sum(X_valid), np.sum(y_valid))
# ===========================================================================
# Building model
# ===========================================================================
X = K.placeholder(shape=(None, MAX_SEQ_LEN), dtype='int32', name='X')
y = K.placeholder(shape=(None, nb_labels), dtype='float32', name='y')
f = N.Sequence([
N.Embedding(tk.nb_words, embedding_dims, W_init=E),
N.Dimshuffle(pattern=(0, 1, 'x', 2)),
N.Conv(num_filters=128, filter_size=(5, 1), strides=1, pad='valid',
activation=K.relu),
N.Pool(pool_size=(5, 1), pad='valid', mode='max'),
N.Conv(num_filters=128, filter_size=(5, 1), strides=1, pad='valid',
activation=K.relu),
N.Pool(pool_size=(5, 1), pad='valid', mode='max'),
N.Conv(num_filters=128, filter_size=(5, 1), strides=1, pad='valid',
(EXP_DIR, MODEL_PATH, LOG_PATH,
TRAIN_PATH, TEST_PATH) = get_model_path('xvec', args)
stdio(LOG_PATH)
# ===========================================================================
# Create data feeder
# ===========================================================================
(train, valid,
test_ids, test_dat,
all_speakers) = prepare_dnn_data(
recipe=args.recipe, feat=FEAT, utt_length=args.l)
n_speakers = len(all_speakers) + 1
# ===========================================================================
# Create the network
# ===========================================================================
inputs = [K.placeholder(shape=(None,) + shape[1:],
dtype='float32',
name='input%d' % i)
for i, shape in enumerate(as_tuple_of_shape(train.shape))]
X = inputs[0]
y = inputs[1]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== the network ====== #
if os.path.exists(MODEL_PATH):
x_vec = N.deserialize(path=MODEL_PATH, force_restore_vars=True)
else:
TRAIN_MODEL = True
with N.args_scope(
['TimeDelayedConv', dict(time_pool='none', activation=K.relu)],
['Dense', dict(activation=K.linear, b_init=None)],
['BatchNorm', dict(activation=K.relu)]
):
X_valid, y_valid = X_train[40000:], y_train[40000:]
X_train, y_train = X_train[:40000], y_train[:40000]
# normalize value to [0, 1]
X_train = X_train / 255.
X_valid = X_valid / 255.
X_test = X_test / 255.
print(ds)
# ====== others ====== #
X_samples, y_samples = X_train[:25], y_train[:25]
input_shape = ds['X_train'].shape
input_ndim = len(input_shape)
print("Train shape:", ctext(X_train.shape, 'cyan'))
print("Valid shape:", ctext(X_valid.shape, 'cyan'))
print("Test shape:", ctext(X_test.shape, 'cyan'))
# ====== create basic tensor ====== #
X = K.placeholder(shape=(None,) + input_shape[1:], name='X_input')
y = K.placeholder(shape=(None,), name='y_input')
# ===========================================================================
# Create the network
# ===========================================================================
LATENT_DROPOUT = 0.3
if args.cnn:
with N.args_scope(([N.Conv, N.Dense], dict(b_init=None, activation=K.linear)),
(N.BatchNorm, dict(activation=tf.nn.elu)),
(N.Pool, dict(mode='max', pool_size=2))):
f_encoder = N.Sequence([
N.Dropout(level=0.5),
N.Dimshuffle((0, 2, 3, 1)) if is_cifar10 else N.Dimshuffle((0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=3, pad='valid'),
N.Pool(),
def _initialize(self, X):
# ====== check inputs dimensions ====== #
if not hasattr(X, 'shape'):
raise ValueError("`X` must have `shape` attribute.")
feat_dim = np.prod(X.shape[1:])
if self._feat_dim is None:
self._feat_dim = feat_dim
# validate input dimension
if feat_dim != self._feat_dim:
raise RuntimeError("Feature dimension mismatch %d and %d" %
(feat_dim, self.feat_dim))
# check if tensorflow op initalized
if hasattr(self, '_f_train'):
return
# ====== binary or multi-classes ====== #
if self.nb_classes == 2:
out_shape = (None,)
fn_activation = tf.nn.sigmoid
fn_loss = tf.losses.sigmoid_cross_entropy
fn_acc = K.metrics.binary_accuracy
else:
out_shape = (None, self.nb_classes)
fn_activation = tf.nn.softmax
fn_loss = tf.losses.softmax_cross_entropy
fn_acc = K.metrics.categorical_accuracy
# ====== create model ====== #
with tf.name_scope(self.name, 'logistic_regression'):
# inputs
self._X = K.placeholder(shape=(None, self.feat_dim),
dtype=self.dtype,
name='%s_input' % self.name)
self._y = K.placeholder(shape=out_shape,
dtype=self.dtype,
name='%s_output' % self.name)
# check the bias
if is_number(self.fit_intercept):
b_init = float(self.fit_intercept)
elif self.fit_intercept is False or \
self.fit_intercept is None:
b_init = None
else:
b_init = self.fit_intercept
# create the model and initialize
with K.variable_dtype(dtype=self.dtype):
self._model = N.Dense(num_units=self.nb_classes,
W_init=init_ops.glorot_uniform_initializer(seed=self._rand_state.randint()),
b_init=b_init,
activation=K.linear)
y_logits = self._model(self._X)
y_prob = fn_activation(y_logits)
# applying class weights
class_weights = tf.constant(value=self._class_weight,
dtype=self.dtype,
name="class_weights")
weights = tf.gather(class_weights,
tf.cast(self._y, 'int32') if self.nb_classes == 2 else
tf.argmax(self._y, axis=-1))
# optimizer
params = [v for v in self._model.variables
if has_roles(v, Weight) or has_roles(v, Bias)]
losses = fn_loss(self._y, y_logits, weights=weights)
l1_norm = tf.norm(self._model.get('W'), ord=1) if self.l1 > 0. else 0
l2_norm = tf.norm(self._model.get('W'), ord=2) if self.l2 > 0. else 0
losses = losses + self.l1 * l1_norm + self.l2 * l2_norm
acc = fn_acc(self._y, y_prob)
updates = self._optimizer.get_updates(losses, params)
# create function
if self.confusion_matrix:
cm = K.metrics.confusion_matrix(y_true=self._y, y_pred=y_prob,
labels=self.nb_classes)
metrics = [losses, acc, cm] if self.confusion_matrix else [losses, acc]
self._f_train = K.function(inputs=(self._X, self._y),
outputs=metrics,
updates=updates,
training=True)
self._f_score = K.function(inputs=(self._X, self._y),
outputs=metrics,
training=False)
self._f_pred_prob = K.function(inputs=self._X,
outputs=y_prob,
training=False)
self._f_pred_logit = K.function(inputs=self._X,
outputs=y_logits,
training=False)
return self
def placeholder(self):
if self.__placeholder is None:
self.__placeholder = K.placeholder(
shape=self.shape, dtype=self.dtype, name=self.name)
return self.__placeholder
).add('--rnn', 'using RNN network', False
).parse()
# ===========================================================================
# Load data
# ===========================================================================
USE_MNIST_DATA = True
if arg.ds.lower() == 'mnist':
ds = F.MNIST_original.get_dataset()
elif arg.ds.lower() == 'fmnist':
ds = F.FMNIST_original.get_dataset()
else:
ds = F.CIFAR10.get_dataset()
USE_MNIST_DATA = False
print(ds)
X = K.placeholder(shape=(None,) + ds['X_train'].shape[1:], name='X')
y = K.placeholder(shape=(None,), name='y', dtype='int32')
y_onehot = tf.one_hot(y, depth=10)
# ===========================================================================
# Build network
# ===========================================================================
if not arg.rnn:
ops = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')) if USE_MNIST_DATA else N.Dimshuffle((0, 2, 3, 1)),
N.BatchNorm(axes='auto'),
N.Conv(32, (3, 3), strides=(1, 1), pad='same',
activation=tf.nn.relu),
N.Pool(pool_size=(2, 2), strides=None),
N.Conv(64, (3, 3), strides=(1, 1), pad='same',
activation=tf.nn.relu),
from __future__ import print_function, division, absolute_import import os os.environ['ODIN'] = 'float32,gpu' import timeit import random import numpy as np from odin.utils import UnitTimer, Progbar from odin import backend as K, nnet as N X1 = K.placeholder(shape=(10000, 1000), name='X1') X2 = K.placeholder(shape=(10000, 1000), name='X2') X3 = K.placeholder(shape=(10000, 2000), name='X3') y1 = K.placeholder(shape=(1000, 2000), name='y1') y2 = K.placeholder(shape=(2000, 3000), name='y2') y3 = K.placeholder(shape=(3000, 4000), name='y3') y4 = K.placeholder(shape=(4000, 5000), name='y4') z = K.dot(X1, y1) + K.dot(X2, y1) z = K.dot(z, y2) z = K.dot(z, y3) z = K.dot(z, y4) print(z) f = K.function([X1, X2, y1, y2, y3, y4], outputs=z) X1 = X3[:, :1000] X2 = X3[:, 1000:]
