def test_simple_rnn(self): np.random.seed(12082518) x = np.random.rand(128, 8, 32) # X = K.placeholder(shape=(None, 8, 32)) X1 = K.placeholder(shape=(None, 8, 32)) X2 = K.placeholder(shape=(None, 8, 32)) X3 = K.placeholder(shape=(None, 8, 33)) f = N.RNN(32, activation=K.relu, input_mode='skip') # y = f(X, mask=K.ones(shape=(128, 8))) graph = K.ComputationGraph(y) self.assertEqual(len(graph.inputs), 1) f1 = K.function([X], y) x1 = f1(x) # ====== different placeholder ====== # y = f(X1) f2 = K.function([X1], y) x2 = f1(x) self.assertEqual(np.sum(x1[0] == x2[0]), np.prod(x1[0].shape)) # ====== pickle load ====== # f = cPickle.loads(cPickle.dumps(f)) y = f(X2) f2 = K.function([X2], y) x3 = f2(x) self.assertEqual(np.sum(x2[0] == x3[0]), np.prod(x2[0].shape)) # ====== other input shape ====== # error_happen = False try: y = f(X3) f3 = K.function([X3], y) x3 = f3(np.random.rand(128, 8, 33)) except (ValueError, Exception): error_happen = True self.assertTrue(error_happen)
def test_rnn_decorator(self): @K.rnn_decorator(sequences='X', states='out') def rnn(X, out): return K.relu(X + out) y = rnn(K.ones(shape=(25, 12, 18, 8)), K.zeros(shape=(25, 18, 8))) f = K.function([], y) self.assertEqual(f()[0].shape, (25, 12, 18, 8))
def test_cudnn_rnn_nnet(self): if get_device() == 'cpu': return print() np.random.seed(1208) batch_size = 6 hidden_size = 4 X_linear = K.placeholder(shape=(None, 3, 8), name='X_linear') X_skip = K.placeholder(shape=(None, 3, hidden_size), name='X_skip') for direction_mode in ['bidirectional', 'unidirectional']: is_bidirectional = direction_mode == 'bidirectional' for nb_layers in [2]: real_layers = nb_layers * 2 if is_bidirectional else nb_layers for rnn_mode in ['gru', 'lstm', 'rnn_relu', 'rnn_tanh']: for init_state, init_state_name in zip( [ None, # None init K.init.uniform, # function init K.variable( np.random.rand(real_layers, 1, hidden_size)), # variable K.variable( np.random.rand(real_layers, batch_size, hidden_size)), # variable K.zeros(shape=(real_layers, 1, hidden_size)), K.ones(shape=(real_layers, batch_size, hidden_size)) ], [ 'None', 'Function', 'Var1', 'VarB', 'Tensor1', 'TensorB' ]): for input_mode in ['linear', 'skip']: if input_mode == 'linear': X = X_linear x = np.random.rand(batch_size, 3, 8) else: X = X_skip x = np.random.rand(batch_size, 3, hidden_size) start = timeit.default_timer() f = N.CudnnRNN(num_units=hidden_size, rnn_mode=rnn_mode, input_mode=input_mode, num_layers=nb_layers, direction_mode=direction_mode, params_split=False, return_states=True) # perform function y = f(X, h0=init_state, c0=init_state) f = K.function(X, y) output = f(x) benchmark = timeit.default_timer() - start self.assertTrue([list(i.shape) for i in output] == [[ batch_size if j is None else j for j in K.get_shape(i) ] for i in y]) print( "*PASSED* [Layers]%s [Mode]%-8s [Input]%-6s [Direction]%-12s [State]%s [Benchmark]%.4f" % (nb_layers, rnn_mode, input_mode, direction_mode, init_state_name, benchmark))
def convolutional_vae(X, saved_states, **kwargs): """ convolutional_vae Return ------ [y_encoder, y_decoder] States ------ [f_inference (encoder), f_generative (decoder)] """ n = kwargs.get('n', 10) batch_size = K.get_shape(X)[0] if batch_size is None: raise ValueError("You must specify batch_size dimension for the input placeholder.") # ====== init ====== # if saved_states is None: # Encoder f_inference = N.Sequence([ N.Reshape(shape=(-1, 28, 28, 1)), N.Conv(num_filters=32, filter_size=3, strides=1, pad='valid', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.Conv(num_filters=64, filter_size=5, strides=2, pad='same', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.Dropout(level=0.1), N.Flatten(outdim=2), N.Dense(num_units=n * 2, b_init=None), N.BatchNorm(axes=0) ], debug=True, name='Encoder') # Decoder f_generative = N.Sequence([ N.Dimshuffle(pattern=(0, 'x', 'x', 1)), N.TransposeConv(num_filters=64, filter_size=3, strides=1, pad='valid', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.TransposeConv(num_filters=32, filter_size=5, strides=2, pad='same', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.TransposeConv(num_filters=1, filter_size=13, strides=3, pad='valid', b_init=None), N.BatchNorm(activation=K.linear), N.Flatten(outdim=3) ], debug=True, name="Decoder") else: f_inference, f_generative = saved_states # ====== Perfrom ====== # # Encoder y_encoder = f_inference(K.cast(X, 'float32')) mu = y_encoder[:, :n] sigma = K.softplus(y_encoder[:, n:]) qz = Normal(mu=mu, sigma=sigma, name='Normal_qz') # Decoder z = Normal(mu=K.zeros(shape=(batch_size, n)), sigma=K.ones(shape=(batch_size, n)), name="Normal_pz") logits = f_generative(z) X_reconstruct = Bernoulli(logits=logits) # inference params = f_inference.parameters + f_generative.parameters inference = ed.KLqp(latent_vars={z: qz}, data={X_reconstruct: X}) # ====== get cost for training ====== # # Bind p(x, z) and q(z | x) to the same placeholder for x. if K.is_training(): import tensorflow as tf inference.initialize() if True: optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) updates = optimizer.apply_gradients( optimizer.compute_gradients(inference.loss, var_list=params)) init = tf.global_variables_initializer() init.run() f_train = K.function(X, inference.loss, updates) else: optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer, var_list=params) init = tf.global_variables_initializer() init.run() f_train = lambda x: inference.update(feed_dict={X: x})['loss'] samples = K.sigmoid(logits) return (samples, z, qz), (f_inference, f_generative)
def convolutional_vae(X, saved_states, **kwargs): """ convolutional_vae Return ------ [y_encoder, y_decoder] States ------ [f_inference (encoder), f_generative (decoder)] """ n = kwargs.get('n', 10) batch_size = K.get_shape(X)[0] if batch_size is None: raise ValueError( "You must specify batch_size dimension for the input placeholder.") # ====== init ====== # if saved_states is None: # Encoder f_inference = N.Sequence([ N.Reshape(shape=(-1, 28, 28, 1)), N.Conv(num_filters=32, filter_size=3, strides=1, pad='valid', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.Conv(num_filters=64, filter_size=5, strides=2, pad='same', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.Dropout(level=0.1), N.Flatten(outdim=2), N.Dense(num_units=n * 2, b_init=None), N.BatchNorm(axes=0) ], debug=True, name='Encoder') # Decoder f_generative = N.Sequence([ N.Dimshuffle(pattern=(0, 'x', 'x', 1)), N.TransposeConv(num_filters=64, filter_size=3, strides=1, pad='valid', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.TransposeConv(num_filters=32, filter_size=5, strides=2, pad='same', b_init=init_ops.constant_initializer(0.), activation=K.elu), N.TransposeConv(num_filters=1, filter_size=13, strides=3, pad='valid', b_init=None), N.BatchNorm(activation=K.linear), N.Flatten(outdim=3) ], debug=True, name="Decoder") else: f_inference, f_generative = saved_states # ====== Perfrom ====== # # Encoder y_encoder = f_inference(K.cast(X, 'float32')) mu = y_encoder[:, :n] sigma = K.softplus(y_encoder[:, n:]) qz = Normal(mu=mu, sigma=sigma, name='Normal_qz') # Decoder z = Normal(mu=K.zeros(shape=(batch_size, n)), sigma=K.ones(shape=(batch_size, n)), name="Normal_pz") logits = f_generative(z) X_reconstruct = Bernoulli(logits=logits) # inference params = f_inference.parameters + f_generative.parameters inference = ed.KLqp(latent_vars={z: qz}, data={X_reconstruct: X}) # ====== get cost for training ====== # # Bind p(x, z) and q(z | x) to the same placeholder for x. if K.is_training(): import tensorflow as tf inference.initialize() if True: optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) updates = optimizer.apply_gradients( optimizer.compute_gradients(inference.loss, var_list=params)) init = tf.global_variables_initializer() init.run() f_train = K.function(X, inference.loss, updates) else: optimizer = tf.train.AdamOptimizer(0.01, epsilon=1.0) inference.initialize(optimizer=optimizer, var_list=params) init = tf.global_variables_initializer() init.run() f_train = lambda x: inference.update(feed_dict={X: x})['loss'] samples = K.sigmoid(logits) return (samples, z, qz), (f_inference, f_generative)