def test_func(func): y = func(x) yT = func.T(func(x)) self.assertEquals(K.eval(y).shape, K.get_shape(y)) self.assertEquals(K.eval(yT).shape, (25, 8, 12)) self.assertEquals(K.eval(yT).shape, K.get_shape(yT))
def _apply(self, X, state=None, memory=None): # time_major: The shape format of the `inputs` and `outputs` Tensors. # If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. # If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. # ====== create attention if necessary ====== # cell = self.cell if self.bidirectional: cell_bw = self.cell_bw # create attention cell if self.attention: if not hasattr(self, "_cell_with_attention"): self._cell_with_attention = self.__attention_creator( cell, X=X, memory=memory) cell = self._cell_with_attention # bidirectional attention if self.bidirectional: if not hasattr(self, "_cell_with_attention_bw"): self._cell_with_attention_bw = self.__attention_creator( cell_bw, X=X, memory=memory) cell_bw = self._cell_with_attention_bw # ====== calling rnn_warpper ====== # ## Bidirectional if self.bidirectional: rnn_func = rnn.bidirectional_dynamic_rnn if self.dynamic \ else rnn.static_bidirectional_rnn state_fw, state_bw = None, None if isinstance(state, (tuple, list)): state_fw = state[0] if len(state) > 1: state_bw = state[1] else: state_fw = state outputs = rnn_func(cell_fw=cell, cell_bw=cell_bw, inputs=X, initial_state_fw=state_fw, initial_state_bw=state_bw, dtype=X.dtype.base_dtype) ## Unidirectional else: rnn_func = rnn.dynamic_rnn if self.dynamic else rnn.static_rnn outputs = rnn_func(cell, inputs=X, initial_state=state, dtype=X.dtype.base_dtype) # ====== initialize cell ====== # if not self._is_initialized_variables: # initialize only once, everytime you call this, the values of # variables changed K.eval(tf.variables_initializer(self.variables)) self._is_initialized_variables = True _infer_variable_role(self.variables) # ====== return ====== # if self.bidirectional: # concat outputs outputs = (tf.concat(outputs[0], axis=-1), outputs[1]) if not self.return_states: return outputs[0] return outputs
def test_save_cudnn_rnn(self): np.random.seed(5218) X = K.variable(np.random.rand(25, 12, 8)) num_layers = 2 num_gates = 'lstm' skip_input = False is_bidirectional = False path = '/tmp/rnn' weights, biases = K.init_rnn(input_dim=8, hidden_dim=18, b_init=init_ops.random_normal_initializer(), num_layers=num_layers, num_gates=num_gates, skip_input=skip_input, is_bidirectional=is_bidirectional) rnn = N.CudnnRNN(num_units=18, W_init=weights, b_init=biases, rnn_mode=num_gates, num_layers=num_layers, skip_input=skip_input, is_bidirectional=is_bidirectional, return_states=False, dropout=0., name="CudnnRNNTest") y = rnn(X) K.initialize_all_variables() y = K.eval(y) N.serialize(nnops=rnn, path=path, binary_output=True, override=True) test_script = r""" from __future__ import print_function, division, absolute_import import os os.environ['ODIN'] = 'gpu,float32,seed=5218' import pickle import numpy as np import tensorflow as tf from tensorflow.python.ops import init_ops from odin.config import randint from odin import backend as K, nnet as N np.random.seed(5218) X = K.variable(np.random.rand(25, 12, 8)) rnn = N.deserialize("%s", force_restore_vars=True) y = rnn(X) K.initialize_all_variables() y = K.eval(y) print(len(rnn.variables), sum(np.sum(K.eval(i)) for i in rnn.variables if K.role.has_roles(i, K.role.Weight)), sum(np.sum(K.eval(i)) for i in rnn.variables if K.role.has_roles(i, K.role.Bias)), y.sum(), (y**2).sum()) """ % path outputs = run_script(test_script)[1] num_variables, w, b, s1, s2 = outputs.split(' ') assert int(num_variables) == len(rnn.variables) assert np.allclose(float(w), sum(np.sum(K.eval(i)) for i in rnn.variables if K.role.has_roles(i, K.role.Weight))) assert np.allclose(float(b), sum(np.sum(K.eval(i)) for i in rnn.variables if K.role.has_roles(i, K.role.Bias))) assert np.allclose(float(s1), y.sum()) assert np.allclose(float(s2), (y**2).sum())
def test_variable_creation(self): np.random.seed(5218) # ====== create by numpy array ====== # tmp = np.random.rand(12, 8).astype('float32') K.variable(value=tmp, dtype='float32', name='x', initialize=True) self.assertTrue(np.all(K.eval(K.variable(name='x')) == tmp)) # ====== create by Variable name ====== # K.variable(value='x', name='z', initialize=True) self.assertTrue(np.all(K.eval(K.variable(name='z')) == tmp)) # ====== create by function ====== # def fn(shape): return np.full(shape=shape, fill_value=8) y = K.variable(value=fn, shape=(12, 18), dtype='float32', name='y', initialize=True) self.assertTrue( np.all(K.eval(y) == np.full(shape=(12, 18), fill_value=8))) # ====== create by initializer ====== # tmp = K.eval(init_ops.orthogonal_initializer(seed=5218)(shape=(8, 8))) w = K.variable(value=init_ops.orthogonal_initializer(seed=5218), shape=(8, 8), dtype='float32', name='w', initialize=True) self.assertTrue(np.all(K.eval(w) == tmp)) # ====== create by number ====== # K.variable(value=25, shape=(8, 8), dtype='float32', name='a', initialize=True) self.assertTrue(K.eval(K.variable(name='a')).sum() == 25 * 8 * 8) # ====== create by tensor ====== # t = tf.constant(value=3, shape=(12, 8), dtype='float32', name='dummy_constant') K.variable(value=t, name='b', initialize=True) self.assertTrue(np.all(K.eval(K.variable(name='b')) == K.eval(t))) # ====== create by Tensor name ====== # K.variable(value='dummy_constant', name='c', initialize=True) self.assertTrue(np.all(K.eval(K.variable(name='c')) == K.eval(t))) # ====== check all variable exist ====== # all_variables = [] all_variables_name = ['x', 'z', 'y', 'w', 'a', 'b', 'c'] for name in all_variables_name: v = K.get_all_variables(name=name) assert len(v) == 1, name all_variables.append(v[0]) # check no duplicate variables self.assertTrue(len(set(all_variables)) == len(all_variables_name))
def _apply(self, X, state=None, memory=None): # time_major: The shape format of the `inputs` and `outputs` Tensors. # If true, these `Tensors` must be shaped `[max_time, batch_size, depth]`. # If false, these `Tensors` must be shaped `[batch_size, max_time, depth]`. # ====== create attention if necessary ====== # cell = self.cell if self.bidirectional: cell_bw = self.cell_bw # create attention cell if self.attention: if not hasattr(self, "_cell_with_attention"): self._cell_with_attention = self.__attention_creator( cell, X=X, memory=memory) cell = self._cell_with_attention # bidirectional attention if self.bidirectional: if not hasattr(self, "_cell_with_attention_bw"): self._cell_with_attention_bw = self.__attention_creator( cell_bw, X=X, memory=memory) cell_bw = self._cell_with_attention_bw # ====== calling rnn_warpper ====== # ## Bidirectional if self.bidirectional: rnn_func = rnn.bidirectional_dynamic_rnn if self.dynamic \ else rnn.static_bidirectional_rnn state_fw, state_bw = None, None if isinstance(state, (tuple, list)): state_fw = state[0] if len(state) > 1: state_bw = state[1] else: state_fw = state outputs = rnn_func(cell_fw=cell, cell_bw=cell_bw, inputs=X, initial_state_fw=state_fw, initial_state_bw=state_bw, dtype=X.dtype.base_dtype) ## Unidirectional else: rnn_func = rnn.dynamic_rnn if self.dynamic else rnn.static_rnn outputs = rnn_func(cell, inputs=X, initial_state=state, dtype=X.dtype.base_dtype) # ====== initialize cell ====== # if not self._is_initialized_variables: # initialize only once, everytime you call this, the values of # variables changed K.eval(tf.variables_initializer(self.variables)) self._is_initialized_variables = True _infer_variable_role(self.variables) # ====== return ====== # if self.bidirectional: # concat outputs outputs = (tf.concat(outputs[0], axis=-1), outputs[1]) if not self.return_states: return outputs[0] return outputs
def test_pool_depool(self): X1 = K.placeholder(shape=(None, 12, 8, 25), name='X1') X2 = K.placeholder(shape=(None, 12, 8, 25, 18), name='X2') x1 = np.random.rand(13, 12, 8, 25) x2 = np.random.rand(13, 12, 8, 25, 18) prog = Progbar(target=2 * 2 * 2 * 3, print_report=True) def check_shape(s1, s2): self.assertEqual(tuple(s1), tuple(s2), msg="%s != %s" % (str(s1), str(s2))) for pool_size in (2, 3): for strides in (2, 3): # strides > window_shape not supported due to inconsistency # between CPU and GPU implementations if pool_size < strides: prog.add(1) continue for pad in ('valid', 'same'): for transpose_mode in ('nn', 'pad_margin', 'repeat'): # ====== print prog ====== # prog['test'] = "Size:%d,Stride:%d,Pad:%s,T:%s" % \ (pool_size, strides, pad, transpose_mode) prog.add(1) # ====== check ops 4D ====== # down = N.Pool(pool_size=pool_size, strides=strides, pad=pad, mode='max', transpose_mode=transpose_mode) up = down.T y1 = down(X1) check_shape( K.eval(y1, { X1: x1 }).shape[1:], y1.shape.as_list()[1:]) y2 = up(y1) check_shape(K.eval(y2, {X1: x1}).shape, x1.shape) # ====== check ops 5D ====== # down = N.Pool(pool_size=pool_size, strides=strides, pad=pad, mode='max', transpose_mode=transpose_mode) up = down.T y1 = down(X2) check_shape( K.eval(y1, { X2: x2 }).shape[1:], y1.shape[1:]) y2 = up(y1) check_shape(K.eval(y2, {X2: x2}).shape, x2.shape)
def test_upsample(self): X = K.variable(np.arange(1, 24 + 1).reshape(2, 2, 3, 2)) self.assertEqual(K.eval(K.sum(X)), 300.) self.assertEqual( K.eval(K.upsample(X, 2, axes=(1, 2), method='nn')).sum(), 1200.) self.assertEqual( K.eval(K.upsample(X, 2, axes=(1, 2), method='pad_margin')).sum(), 300.) self.assertEqual( K.eval(K.upsample(X, 2, axes=(1, 2), method='repeat')).sum(), 1200.)
def test_linear_algebra_value(self): np.random.seed(1208) x = K.variable(np.random.randn(2, 4, 3)) y = K.variable(np.random.rand(1, 2, 3, 5)) z = K.dot(x, y) self.assertEqual(K.get_shape(z), (2, 4, 1, 2, 5)) self.assertEqual( repr(np.sum(K.eval(z)))[:8], "-1.0198305134529524"[:8]) np.random.seed(1208) x = K.variable(np.random.randn(100, 3, 4, 5)) y = K.variable(np.random.rand(100, 12, 5, 6)) z = K.batched_dot(x, y) self.assertEqual(K.get_shape(z), K.eval(z).shape) self.assertEqual(repr(K.eval(z).sum())[:7], "1655.44")
def test_confusion_matrix(self): from sklearn.metrics import confusion_matrix y1 = np.random.randint(0, 8, size=100) y2 = np.random.randint(0, 8, size=100) y_pred = K.variable(y1) y_true = K.variable(y2) confusion = K.confusion_matrix(y_pred, y_true) r1 = K.eval(confusion) r2 = confusion_matrix(y1, y2) self.assertEqual(np.sum(r1 - r2), 0.)
def _initialize_param(name, spec, shape): """ return a ndarray or trainable_variable """ ##################################### # 0. initializing function. if callable(spec): spec = spec(shape) ##################################### # 1. Shared variable, just check the shape. if K.is_trainable_variable(spec): spec_shape = K.get_shape(spec) if not isinstance(spec_shape, tuple): spec_shape = K.eval(spec_shape) if shape is None: shape = spec_shape elif tuple(shape) != tuple(spec_shape): raise Exception( 'Given variable has different shape from requirement' ', %s != %s' % (str(spec_shape), str(shape))) ##################################### # 2. expression, we can only check number of dimension. elif K.is_variable(spec): # We cannot check the shape here, Theano expressions (even shared # variables) do not have a fixed compile-time shape. We can check the # dimensionality though. # Note that we cannot assign a name here. We could assign to the # `name` attribute of the variable, but the user may have already # named the variable and we don't want to override this. if shape is not None and K.ndim(spec) != len(shape): raise Exception( "parameter with name=%s has %d dimensions, should be " "%d" % (name, spec.ndim, len(shape))) ##################################### # 3. numpy ndarray, create shared variable wraper for it. elif isinstance(spec, np.ndarray): if shape is not None and spec.shape != shape: raise RuntimeError( "parameter with name=%s has shape %s, should be " "%s" % (name, spec.shape, shape)) ##################################### # 5. Exception. else: raise RuntimeError("cannot initialize parameters: 'spec' is not " "a numpy array, a Theano expression, or a " "callable") return spec, shape
def test_randomization(self): # same rng generate the same random K.set_rng(12) a1 = K.eval(K.random_normal(shape=(10, 10), mean=0, std=1)) b1 = K.eval(K.random_uniform(shape=(10, 10), low=-1, high=1)) c1 = K.eval(K.random_binomial(shape=(10, 10), p=0.7)) K.set_rng(12) a2 = K.eval(K.random_normal(shape=(10, 10), mean=0, std=1)) b2 = K.eval(K.random_uniform(shape=(10, 10), low=-1, high=1)) c2 = K.eval(K.random_binomial(shape=(10, 10), p=0.7)) self.assertTrue(np.array_equal(a1, a2)) self.assertTrue(np.array_equal(b1, b2)) self.assertTrue(np.array_equal(c1, c2)) self.assertTrue(np.abs(np.mean(a1)) < 0.1) self.assertTrue(np.std(a1) >= 0.8 and np.std(a1) <= 1.2) self.assertTrue(np.sum(c1) <= 80) # direct random generation a = K.eval(K.random_normal(shape=(10, 10))) b = K.eval(K.random_uniform(shape=(10, 10))) c = K.eval(K.random_binomial(shape=(10, 10), p=0.1)) self.assertTrue(np.sum(c) <= 20)
def test_computational_graph2(self): np.random.seed(1208) X = K.variable(np.zeros((8, 12)), name='X') Y = K.variable(np.random.rand(12, 8), name='Y') Z = K.placeholder(shape=(8, 8), name='Z') a = K.dot(X, Y) add_roles(a, Auxiliary) a = a + Z g1 = K.ComputationGraph(a) self.assertEqual(len(g1.trainable_variables), 2) self.assertEqual(len(g1.placeholders), 1) self.assertEqual(len(g1.updates), 1) self.assertEqual(len(g1.auxiliary_variables), 1) f = K.function(Z, [a] + g1.auxiliary_variables) output = f(np.random.rand(8, 8)) self.assertEqual(repr(np.sum(output[0]))[:5], "32.20") self.assertEqual(np.sum(output[1]), 0) self.assertEqual(np.unique(K.eval(X)).tolist(), [12.])
def show_image(x, is_probability=False): from matplotlib import pyplot as plt from odin import backend as K x = anything2image(x) if x.shape[0] > 32: x = tf.nn.max_pool(np.reshape(x, (1, x.shape[0], x.shape[1], 1)), ksize=(1, 4, 4, 1), strides=(1, 4, 4, 1), padding="SAME") x = np.squeeze(K.eval(x)) ax = plt.gca() plt.imshow(x, interpolation='nearest', cmap=plt.cm.Greys_r, vmin=0. if is_probability else None, vmax=1. if is_probability else None) plt.xticks(()) ax.set_xticklabels([]) plt.yticks(()) ax.set_yticklabels([]) ax.set_aspect(aspect='auto')
def test_auto_infer_shape(self): x = K.variable(np.random.rand(8, 25, 12)) y = K.placeholder((None, 25, 12)) def test_func(func): self.assertEquals(K.get_shape(func(x, 0)), K.eval(func(x, 0)).shape) self.assertEquals(K.get_shape(func(x, -1)), K.eval(func(x, -1)).shape) self.assertEquals(K.get_shape(func(x, 1, True)), K.eval(func(x, 1, True)).shape) self.assertEquals(K.get_shape(func(x, 0)), K.get_shape(func(y, 0))) self.assertEquals(K.get_shape(func(x, 0, True)), K.get_shape(func(y, 0, True))) if func != K.argmax and func != K.argmin: self.assertEquals(K.get_shape(func(x, (1, -1))), K.eval(func(x, (1, -1))).shape) self.assertEquals(K.get_shape(func(x, (0, 1))), K.eval(func(x, (0, 1))).shape) self.assertEquals(K.get_shape(func(x, (0, 1), True)), K.eval(func(x, (0, 1), True)).shape) test_func(K.var) test_func(K.max) test_func(K.min) test_func(K.any) test_func(K.sum) test_func(K.prod) test_func(K.mean) test_func(K.std) test_func(K.any) test_func(K.argmax) test_func(K.argmin) self.assertEquals(K.get_shape(K.argsort(x)), K.eval(K.argsort(x)).shape)
def test_func(func): self.assertEquals(K.get_shape(func(x, 0)), K.eval(func(x, 0)).shape) self.assertEquals(K.get_shape(func(x, -1)), K.eval(func(x, -1)).shape) self.assertEquals(K.get_shape(func(x, 1, True)), K.eval(func(x, 1, True)).shape) self.assertEquals(K.get_shape(func(x, 0)), K.get_shape(func(y, 0))) self.assertEquals(K.get_shape(func(x, 0, True)), K.get_shape(func(y, 0, True))) if func != K.argmax and func != K.argmin: self.assertEquals(K.get_shape(func(x, (1, -1))), K.eval(func(x, (1, -1))).shape) self.assertEquals(K.get_shape(func(x, (0, 1))), K.eval(func(x, (0, 1))).shape) self.assertEquals(K.get_shape(func(x, (0, 1), True)), K.eval(func(x, (0, 1), True)).shape)
def test_ops(self): x = K.variable(np.random.rand(8, 12)) y = K.variable(np.random.rand(12, 25)) z = K.placeholder((25, 18, 13)) w = K.placeholder((18, 18)) # ====== dot ====== # t = K.dot(x, y) self.assertEquals(K.get_shape(t), (8, 25)) self.assertEquals(K.get_shape(t), K.eval(t).shape) t = K.dot(t, K.dimshuffle(z, (1, 0, 2))) self.assertEquals(K.get_shape(t), (8, 18, 13)) # ====== transpose ====== # self.assertEquals(K.get_shape(K.transpose(z)), (13, 18, 25)) self.assertEquals(K.get_shape(K.transpose(t, axes=(2, 0, 1))), (13, 8, 18)) # ====== eye ====== # self.assertEquals(K.get_shape(K.eye(5)), K.eval(K.eye(5)).shape) # ====== diag ====== # self.assertEquals(K.get_shape(K.diag(w)), (18, )) # self.assertEquals(K.get_shape(K.diag(x)), # K.eval(K.diag(y)).shape) self.assertEquals(K.get_shape(K.square(x)), K.eval(K.square(x)).shape) self.assertEquals(K.get_shape(K.abs(x)), K.eval(K.abs(x)).shape) self.assertEquals(K.get_shape(K.sqrt(x)), K.eval(K.sqrt(x)).shape) self.assertEquals(K.get_shape(K.exp(x)), K.eval(K.exp(x)).shape) self.assertEquals(K.get_shape(K.log(x)), K.eval(K.log(x)).shape) self.assertEquals(K.get_shape(K.round(x)), K.eval(K.round(x)).shape) self.assertEquals(K.get_shape(K.pow(x, 2)), K.eval(K.pow(x, 2)).shape) self.assertEquals(K.get_shape(K.clip(x, -1, 1)), K.eval(K.clip(x, -1, 1)).shape) self.assertEquals(K.get_shape(K.inv(x)), K.eval(K.inv(x)).shape)
def test_cudnn_rnn(self): if get_ngpu() == 0: return print() batch_size = 2 time_steps = 5 input_dim = 12 hidden_dim = 8 X = K.variable(value=np.random.rand(batch_size, time_steps, input_dim), dtype='float32', name='X') for rnn_mode in ('lstm', 'rnn_relu', 'gru'): for num_layers in [1, 2]: for W_init in [ init_ops.glorot_uniform_initializer(seed=1234), init_ops.random_normal_initializer(seed=1234) ]: for b_init in [0, 1]: for bidirectional in (True, False): for skip_input in (False, ): print('RNNmode:%s' % rnn_mode, "#Layers:%d" % num_layers, 'Bidirectional:%s' % bidirectional, 'SkipInput:%s' % skip_input) weights, biases = K.init_rnn( input_dim=input_dim, hidden_dim=hidden_dim, num_gates=rnn_mode, num_layers=num_layers, W_init=W_init, b_init=b_init, skip_input=skip_input, cudnn_vector=False, is_bidirectional=bidirectional, name=None) # ====== check number of params ====== # params1 = K.params_to_cudnn(weights, biases) n = params1.shape[0].value nb_params = cudnn_rnn_ops.cudnn_rnn_opaque_params_size( rnn_mode=rnn_mode, num_layers=num_layers, num_units=hidden_dim, input_size=input_dim, input_mode='skip_input' if skip_input else 'linear_input', direction='bidirectional' if bidirectional else 'unidirectional') nb_params = K.eval(nb_params) assert n == nb_params # ====== check cannonical shape match ====== # kwargs = { 'num_layers': num_layers, 'num_units': hidden_dim, 'input_mode': 'skip_input' if skip_input else 'linear_input', 'direction': 'bidirectional' if bidirectional else 'unidirectional' } if rnn_mode == 'lstm': rnn = cudnn_rnn.CudnnLSTM(**kwargs) elif rnn_mode == 'gru': rnn = cudnn_rnn.CudnnGRU(**kwargs) if rnn_mode == 'rnn_relu': rnn = cudnn_rnn.CudnnRNNRelu(**kwargs) if rnn_mode == 'rnn_tanh': rnn = cudnn_rnn.CudnnRNNTanh(**kwargs) rnn.build(input_shape=(None, None, input_dim)) assert len(weights) == len( rnn.canonical_weight_shapes) assert len(biases) == len( rnn.canonical_bias_shapes) for w, s in zip(weights, rnn.canonical_weight_shapes): assert tuple(w.shape.as_list()) == s # ====== check params conversion ====== # K.initialize_all_variables() params2 = cudnn_rnn_ops.cudnn_rnn_canonical_to_opaque_params( rnn_mode=rnn_mode, num_layers=num_layers, num_units=hidden_dim, input_size=input_dim, input_mode='skip_input' if skip_input else 'linear_input', direction='bidirectional' if bidirectional else 'unidirectional', weights=weights, biases=biases) assert np.all( K.eval(params1) == K.eval(params2)) # ====== odin cudnn implementation ====== # name = 'TEST' + uuid(length=25) outputs = K.cudnn_rnn( X=X, num_units=hidden_dim, rnn_mode=rnn_mode, num_layers=num_layers, parameters=None, skip_input=skip_input, is_bidirectional=bidirectional, dropout=0.1, name=name) K.initialize_all_variables() s0 = K.eval(outputs[0]).sum() s1 = K.eval(outputs[1]).sum() all_variables = K.get_all_variables(scope=name) new_weights = [ i for i in all_variables if K.role.has_roles(i, roles=K.role.Weight) ] new_biases = [ i for i in all_variables if K.role.has_roles(i, roles=K.role.Bias) ] new_weights, new_biases = K.sort_cudnn_params( new_weights, new_biases, rnn_mode=rnn_mode) assert len(weights) == len(weights) assert len(biases) == len(biases) for i, j in zip(weights + biases, new_weights + new_biases): assert i.name.split( '/')[-1] == j.name.split('/')[-1] # ====== CudnnRNN wrapper ====== # rnn = N.CudnnRNN( num_units=hidden_dim, W_init=new_weights, b_init=new_biases, rnn_mode=rnn_mode, num_layers=num_layers, skip_input=skip_input, is_bidirectional=bidirectional, return_states=True, dropout=0.) outputs = rnn(X) K.initialize_all_variables() y0 = K.eval(outputs[0]).sum() y1 = K.eval(outputs[1]).sum() assert y0 == s0 assert y1 == s1
def test_func(func): y = func(x) yT = func.T(func(x)) self.assertEquals(K.eval(y).shape, tuple(y.shape.as_list())) self.assertEquals(K.eval(yT).shape, (25, 8, 12)) self.assertEquals(K.eval(yT).shape, tuple(yT.shape.as_list()))
update_ops = K.optimizers.Adam(lr=0.001).minimize(loss) K.initialize_all_variables() # ====== intitalize ====== # record_train_loss = [] record_valid_loss = [] patience = 3 epoch = 0 # We want the rate to go up but the distortion to go down while True: # ====== training ====== # train_losses = [] prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch) start_time = timeit.default_timer() for start, end in batching(batch_size=args.bs, n=X_train.shape[0], seed=K.get_rng().randint(10e8)): _ = K.eval(loss, feed_dict={X: X_train[start:end]}, update_after=update_ops) prog.add(end - start) train_losses.append(_) # ====== training log ====== # print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time)) print("[Training set] Loss: %.4f" % np.mean(train_losses)) # ====== validation set ====== # code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid}) print("[Valid set] Loss: %.4f" % lo) # ====== record the history ====== # record_train_loss.append(np.mean(train_losses)) record_valid_loss.append(lo) # ====== plotting ====== # if args.dim > 2: code_samples = ml.fast_pca(code_samples, n_components=2, random_state=K.get_rng().randint(10e8))
def test_func(x, y, func): self.assertEquals(K.get_shape(func(x, y)), K.eval(func(x, y)).shape)
def test_func(x, func, *args, **kwargs): self.assertEquals(K.get_shape(func(x, *args, **kwargs)), K.eval(func(x, *args, **kwargs)).shape)
update_ops = K.optimizers.Adam(lr=0.001).minimize(loss) K.initialize_all_variables() # ====== intitalize ====== # record_train_loss = [] record_valid_loss = [] patience = 3 epoch = 0 # We want the rate to go up but the distortion to go down while True: # ====== training ====== # train_losses = [] prog = Progbar(target=X_train.shape[0], name='Epoch%d' % epoch) start_time = timeit.default_timer() for start, end in batching(batch_size=args.bs, n=X_train.shape[0], seed=K.get_rng().randint(10e8)): _ = K.eval(loss, feed_dict={X: X_train[start:end]}, update_after=update_ops) prog.add(end - start) train_losses.append(_) # ====== training log ====== # print(ctext("[Epoch %d]" % epoch, 'yellow'), '%.2f(s)' % (timeit.default_timer() - start_time)) print("[Training set] Loss: %.4f" % np.mean(train_losses)) # ====== validation set ====== # code_samples, lo = K.eval([Z, loss], feed_dict={X: X_valid}) print("[Valid set] Loss: %.4f" % lo) # ====== record the history ====== # record_train_loss.append(np.mean(train_losses)) record_valid_loss.append(lo) # ====== plotting ====== # if args.dim > 2: code_samples = ml.fast_pca(code_samples, n_components=2, random_state=K.get_rng().randint(10e8))
def test_basic_ops_value(self): np.random.seed(12082518) x = K.variable(np.random.randn(8, 8)) y = K.variable(np.random.randn(8, 8)) z = K.variable(np.random.randint(0, 2, size=(8, 8)), dtype=np.bool) w = K.variable(np.random.randint(0, 2, size=(8, 8)), dtype=np.bool) self.assertEqual(round(np.sum(K.eval(K.relu(x, alpha=0.12))) * 10000), 276733) self.assertEqual(round(np.sum(K.eval(K.elu(x, alpha=0.12))) * 10000), 289202) self.assertEqual(np.sum(K.eval(K.softmax(x))), 8.0) self.assertEqual(round(np.sum(K.eval(K.softplus(x))) * 10000), 554564) self.assertEqual(round(np.sum(K.eval(K.softsign(x))) * 100000), 211582) self.assertEqual(round(np.sum(K.eval(K.sigmoid(x))) * 10000), 330427) self.assertEqual(round(np.sum(K.eval(K.hard_sigmoid(x))) * 10000), 330836) self.assertEqual(round(np.sum(K.eval(K.tanh(x))) * 100000), 290165) self.assertEqual(round(np.sum(K.eval(K.square(x))) * 10000), 744492) self.assertEqual(round(np.sum(K.eval(K.sqrt(x))) * 10000), 300212) self.assertEqual(round(np.sum(K.eval(K.abs(x))) * 10000), 559979) self.assertEqual(np.sum(K.eval(K.sign(x))), 6.0) self.assertEqual(round(np.sum(K.eval(K.inv(x))) * 1000), 495838) self.assertEqual(round(np.sum(K.eval(K.exp(x))) * 1000), 122062) self.assertEqual(round(np.sum(K.eval(K.log(K.abs(x)))) * 10000), -344491) self.assertEqual(np.sum(K.eval(K.round(x))), 5.0) self.assertEqual(round(np.sum(K.eval(K.pow(x, 8))) * 100), 398153) self.assertEqual( round(np.sum(K.eval(K.clip(x, -0.12, 0.12))) * 1000000), 620529) # TODO: pygpu (libgpuarray) still not support diag # self.assertEqual(round(np.sum(K.eval(K.diag(x))) * 100000), 325289) self.assertEqual(np.sum(K.eval(K.eye(12, 8))), 8.0) self.assertEqual(np.sum(K.eval(K.eq(z, w))), 38) self.assertEqual(np.sum(K.eval(K.neq(z, w))), 26) self.assertEqual(np.sum(K.eval(K.gt(x, y))), 33) self.assertEqual(np.sum(K.eval(K.ge(x, y))), 33) self.assertEqual(np.sum(K.eval(K.lt(x, y))), 31) self.assertEqual(np.sum(K.eval(K.le(x, y))), 31) self.assertEqual(round(np.sum(K.eval(K.switch(z, x, y))) * 100000), 139884)