def add_encoder_layer(self, input, name, training, layer_to_skip_connect, local_inner_layers, num_features, dim_reduce=False, dropout_rate=0.0): [b1, h1, w1, d1] = input.get_shape().as_list() if layer_to_skip_connect is not None: [b0, h0, w0, d0] = layer_to_skip_connect.get_shape().as_list() if h0 > h1: skip_connect_layer = self.conv_layer(layer_to_skip_connect, int(layer_to_skip_connect.get_shape()[3]), [3, 3], strides=(2, 2)) else: skip_connect_layer = layer_to_skip_connect else: skip_connect_layer = layer_to_skip_connect current_layers = [input, skip_connect_layer] current_layers.extend(local_inner_layers) current_layers = remove_duplicates(current_layers) outputs = tf.concat(current_layers, axis=3) if dim_reduce: outputs = self.conv_layer(outputs, num_features, [3, 3], strides=(2, 2)) outputs = leaky_relu(features=outputs) outputs = layer_norm(inputs=outputs, center=True, scale=True) # outputs = tf.nn.relu(outputs) # outputs = tf.layers.batch_normalization(outputs, training=training, momentum=0.9) outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) else: outputs = self.conv_layer(outputs, num_features, [3, 3], strides=(1, 1)) outputs = leaky_relu(features=outputs) outputs = layer_norm(inputs=outputs, center=True, scale=True) # outputs = tf.nn.relu(outputs) # outputs = tf.layers.batch_normalization(outputs, training=training, momentum=0.9) return outputs
def testName(self): np_values = np.array([-2, -1, 0, 1, 2], dtype=np.float64) outputs_with_name_set = nn_ops.leaky_relu( constant_op.constant(np_values), name='test_relu_op') self.assertEqual(outputs_with_name_set.name, 'test_relu_op:0') outputs_without_name_set = nn_ops.leaky_relu( constant_op.constant(np_values)) self.assertEqual(outputs_without_name_set.name, 'LeakyRelu:0')
def testName(self): np_values = np.array([-2, -1, 0, 1, 2], dtype=np.float64) outputs_with_name_set = nn_ops.leaky_relu( constant_op.constant(np_values), name='test_relu_op') self.assertEqual(outputs_with_name_set.name, 'test_relu_op:0') outputs_without_name_set = nn_ops.leaky_relu( constant_op.constant(np_values)) self.assertEqual(outputs_without_name_set.name, 'LeakyRelu:0')
def add_encoder_layer(self, input, name, training, layer_to_skip_connect, local_inner_layers, num_features, dim_reduce=False, dropout_rate=0.0): """ Adds a resnet encoder layer. :param input: The input to the encoder layer :param training: Flag for training or validation :param dropout_rate: A float or a placeholder for the dropout rate :param layer_to_skip_connect: Layer to skip-connect this layer to :param local_inner_layers: A list with the inner layers of the current Multi-Layer :param num_features: Number of feature maps for the convolutions :param dim_reduce: Boolean value indicating if this is a dimensionality reducing layer or not :return: The output of the encoder layer :return: """ [b1, h1, w1, d1] = input.get_shape().as_list() if layer_to_skip_connect is not None: [b0, h0, w0, d0] = layer_to_skip_connect.get_shape().as_list() if h0 > h1: skip_connect_layer = self.conv_layer( layer_to_skip_connect, int(layer_to_skip_connect.get_shape()[3]), [1, 1], strides=(2, 2)) else: skip_connect_layer = layer_to_skip_connect else: skip_connect_layer = layer_to_skip_connect current_layers = [input, skip_connect_layer] current_layers.extend(local_inner_layers) current_layers = remove_duplicates(current_layers) outputs = tf.concat(current_layers, axis=3) if dim_reduce: outputs = self.conv_layer(outputs, num_features, [3, 3], strides=(2, 2)) outputs = leaky_relu(features=outputs) outputs = layer_norm(inputs=outputs, center=True, scale=True) outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) else: outputs = self.conv_layer(outputs, num_features, [3, 3], strides=(1, 1)) outputs = leaky_relu(features=outputs) outputs = layer_norm(inputs=outputs, center=True, scale=True) return outputs
def testUnexpectedAlphaValue(self): self.assertAllClose( np.array([[-9.0, 0.7, -5.0, 0.3, -0.1], [0.1, -3.0, 0.5, -27.0, 0.9]]), nn_ops.leaky_relu(np.array([[-0.9, 0.7, -0.5, 0.3, -0.01], [0.1, -0.3, 0.5, -2.7, 0.9]]), alpha=10)) self.assertAllClose( np.array([[9.0, 0.7, 5.0, 0.3, 0.1], [0.1, 3.0, 0.5, 27.0, 0.9]]), nn_ops.leaky_relu(np.array([[-0.9, 0.7, -0.5, 0.3, -0.01], [0.1, -0.3, 0.5, -2.7, 0.9]]), alpha=-10))
def _testLeakyRelu(self, np_features, alpha, use_gpu=False): np_leaky_relu = self._npLeakyRelu(np_features, alpha) with self.test_session(use_gpu=use_gpu): leaky_relu = nn_ops.leaky_relu(np_features, alpha) tf_leaky_relu = leaky_relu.eval() self.assertAllClose(np_leaky_relu, tf_leaky_relu) self.assertShapeEqual(np_leaky_relu, leaky_relu)
def _testLeakyRelu(self, np_features, alpha, use_gpu=False): np_leaky_relu = self._npLeakyRelu(np_features, alpha) with self.test_session(use_gpu=use_gpu): leaky_relu = nn_ops.leaky_relu(np_features, alpha) tf_leaky_relu = leaky_relu.eval() self.assertAllClose(np_leaky_relu, tf_leaky_relu) self.assertShapeEqual(np_leaky_relu, leaky_relu)
def conv_block(input, phase, convs, do_skip=True): x = input count = 0 for conv in convs: if count == (len(convs) - 2) and do_skip: skip_connection = x count += 1 if conv['stride'] > 1: x = ZeroPadding2D(((1, 0), (1, 0)))(x) x = Conv2D(conv['filter'], conv['kernel'], strides=conv['stride'], padding='valid' if conv['stride'] > 1 else 'same', name='conv_' + str(conv['layer_idx']), use_bias=False if conv['norm'] else True)(x) if conv['norm']: x = batch_normalization(inputs=x, training=phase, name='norm_' + str(conv['layer_idx'])) if conv['leaky']: x = leaky_relu(x, alpha=0.1) return add([skip_connection, x]) if do_skip else x
def __call__(self, image_input, training=False, dropout_rate=0.0): """ Runs the CNN producing the embeddings and the gradients. :param image_input: Image input to produce embeddings for. [batch_size, 28, 28, 1] :param training: A flag indicating training or evaluation :param dropout_rate: A tf placeholder of type tf.float32 indicating the amount of dropout applied :return: Embeddings of size [batch_size, 64] """ with tf.variable_scope(self.name, reuse=self.reuse): outputs = image_input with tf.variable_scope('conv_layers'): for idx, num_filters in enumerate(self.layer_sizes): with tf.variable_scope('g_conv_{}'.format(idx)): if idx == len(self.layer_sizes) - 1: outputs = tf.layers.conv2d(outputs, num_filters, [2, 2], strides=(1, 1), padding='VALID') else: outputs = tf.layers.conv2d(outputs, num_filters, [3, 3], strides=(1, 1), padding='VALID') outputs = leaky_relu(outputs) outputs = tf.contrib.layers.batch_norm(outputs, updates_collections=None, decay=0.99, scale=True, center=True, is_training=training) outputs = max_pool(outputs, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) image_embedding = tf.contrib.layers.flatten(outputs) self.reuse = tf.AUTO_REUSE self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return image_embedding
def testValues(self): for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype) outputs = nn_ops.leaky_relu(constant_op.constant(np_values)) with self.test_session() as sess: outputs = sess.run(outputs) tol = 2e-3 if dtype == np.float16 else 1e-6 self.assertAllClose( outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol)
def testValues(self): for dtype in [np.int32, np.int64, np.float16, np.float32, np.float64]: np_values = np.array([-2, -1, 0, 1, 2], dtype=dtype) outputs = nn_ops.leaky_relu(constant_op.constant(np_values)) with self.cached_session() as sess: outputs = sess.run(outputs) tol = 2e-3 if dtype == np.float16 else 1e-6 self.assertAllClose( outputs, [-0.4, -0.2, 0.0, 1.0, 2.0], rtol=tol, atol=tol)
def __call__(self, image_input, training=False, dropout_rate=0.0): """ Runs the CNN producing the embeddings and the gradients. :param image_input: Image input to produce embeddings for. [batch_size, 28, 28, 1] :param training: A flag indicating training or evaluation :param dropout_rate: A tf placeholder of type tf.float32 indicating the amount of dropout applied :return: Embeddings of size [batch_size, 64] """ with tf.variable_scope(self.name, reuse=self.reuse): outputs = image_input #image_input.shape: (32,28,28,1) with tf.variable_scope('conv_layers'): for idx, num_filters in enumerate(self.layer_sizes): with tf.variable_scope('g_conv_{}'.format(idx)): if idx == len(self.layer_sizes) - 1: outputs = tf.layers.conv2d(outputs, num_filters, [2, 2], strides=(1, 1), padding='VALID') else: outputs = tf.layers.conv2d(outputs, num_filters, [3, 3], strides=(1, 1), padding='VALID') outputs = leaky_relu(outputs) outputs = tf.contrib.layers.batch_norm( outputs, updates_collections=None, decay=0.99, scale=True, center=True, is_training=training) outputs = max_pool(outputs, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME') #outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) # # 全连接层1 # W_fc1 = weight_variable([64,1024]) # b_fc1 = bias_variable([1024]) # h_pool2_flat = tf.reshape(outputs, [-1,7*7*64]) #[n_samples,1,1,64]->>[n_samples,1*1*64] # h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) # h_fc1_drop = tf.nn.dropout(h_fc1, dropout_rate) # 减少计算量dropout # # 全连接层2 # W_fc2 = weight_variable([1024, 所有标签种类数]) # b_fc2 = bias_variable([所有标签种类数]) # prediction = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 image_embedding = tf.contrib.layers.flatten( outputs) #image_embedding: (32,64) self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) return image_embedding
def testGradientScalar(self): with self.test_session() as sess: x = variables.Variable(-100.) y = nn_ops.leaky_relu(x, 0.05) loss = y**2 optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) train_op = optimizer.minimize(loss) sess.run(variables.global_variables_initializer()) sess.run(train_op) self.assertAllClose(x.eval(), -99.9)
def testGradientScalar(self): with self.test_session() as sess: x = variables.Variable(-100.) y = nn_ops.leaky_relu(x, 0.05) loss = y**2 optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=0.2) train_op = optimizer.minimize(loss) sess.run(variables.global_variables_initializer()) sess.run(train_op) self.assertAllClose(x.eval(), -99.9)
def __call__(self, image_input, training=False, dropout_rate=0.0): """ Runs the CNN producing the predictions and the gradients. :param image_input: Image input to produce embeddings for. e.g. for EMNIST [batch_size, 28, 28, 1] :param training: A flag indicating training or evaluation :param dropout_rate: A tf placeholder of type tf.float32 indicating the amount of dropout applied :return: Embeddings of size [batch_size, self.num_classes] """ with tf.variable_scope(self.name, reuse=self.reuse): layer_features = [] with tf.variable_scope('FCCLayerNet'): outputs = image_input for i in range(len(self.layer_stage_sizes)): with tf.variable_scope('conv_stage_{}'.format(i)): for j in range(self.inner_layer_depth): with tf.variable_scope('conv_{}_{}'.format(i, j)): outputs = tf.layers.dense( outputs, units=self.layer_stage_sizes[i]) outputs = leaky_relu( outputs, name="leaky_relu{}".format(i)) layer_features.append(outputs) if self.batch_norm_use: outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=False) outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) # apply dropout only at dimensionality # reducing steps, i.e. the last layer in # every group c_conv_encoder = outputs c_conv_encoder = tf.contrib.layers.flatten(c_conv_encoder) c_conv_encoder = tf.layers.dense(c_conv_encoder, units=self.num_classes) self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) if not self.build_completed: self.build_completed = True count_parameters(self.variables, "FCCLayerNet") return c_conv_encoder, layer_features
def testGradientFloat32(self): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], shape=[2, 5], name="x") y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") x_init = np.asarray( [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], dtype=np.float32, order="F") err = gradient_checker.compute_gradient_error( x, [2, 5], y, [2, 5], x_init_value=x_init) print("leaky_relu (float32) gradient err = ", err) self.assertLess(err, 1e-4)
def testRange(self): batch_size = 3 height, width = 4, 4 np.random.seed(1) # Make it reproducible. inputs = np.random.uniform( size=(batch_size, height, width, 3)).astype(np.float32) inputs = constant_op.constant(inputs) outputs = nn_ops.leaky_relu(inputs) self.assertEquals(inputs.shape, outputs.shape) with self.test_session() as sess: inputs, outputs = sess.run([inputs, outputs]) self.assertGreaterEqual(outputs.min(), 0.0) self.assertLessEqual(outputs.max(), 1.0) self.assertAllClose(inputs, outputs)
def testGradientFloat32(self): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], shape=[2, 5], name="x") y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") x_init = np.asarray( [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], dtype=np.float32, order="F") err = gradient_checker.compute_gradient_error( x, [2, 5], y, [2, 5], x_init_value=x_init) print("leaky_relu (float32) gradient err = ", err) self.assertLess(err, 1e-4)
def __call__(self, image_input, scope, training=False, dropout_rate=0.0): outputs = image_input encoder_layers = [] current_layers = [outputs] with tf.variable_scope(scope): for i, layer_size in enumerate(self.layer_sizes): encoder_inner_layers = [outputs] with tf.variable_scope('g_conv{}'.format(i)): if i == 0: outputs = self.conv_layer(outputs, num_filters=64, filter_size=(3, 3), strides=(2, 2), scope='g_conv{}'.format(i)) outputs = leaky_relu(features=outputs) outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=True, scope='bn_1') current_layers.append(outputs) encoder_inner_layers.append(outputs) else: for j in range(self.inner_layers[i]): # Build the inner Layers of the MultiLayer with tf.variable_scope('g_conv_inner_layer{}'.format(j)): outputs = self.add_encoder_layer(input=outputs, training=training, name="encoder_layer_{}_{}".format(i, j), layer_to_skip_connect=current_layers, num_features=self.layer_sizes[i], dim_reduce=False, local_inner_layers=encoder_inner_layers, dropout_rate=dropout_rate, scope="encoder_layer_{}_{}".format(i, j)) encoder_inner_layers.append(outputs) # current_layers.append(outputs) # add final dim reducing conv layer for this MultiLayer outputs = self.add_encoder_layer(input=outputs, name="encoder_layer_{}".format(j), training=training, layer_to_skip_connect=current_layers, local_inner_layers=encoder_inner_layers, num_features=self.layer_sizes[i], dim_reduce=True, dropout_rate=dropout_rate, scope="encoder_layer_{}".format(i)) current_layers.append(outputs) encoder_layers.append(outputs) # print('{}_th encoder output', outputs) return outputs, encoder_layers
def testGradGradFloat32(self): with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], shape=[2, 5], name="x") y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") z = gradients_impl.gradients(y, x) x_init = np.asarray([[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], dtype=np.float32, order="F") err = gradient_checker.compute_gradient_error( x, [2, 5], z[0], [2, 5], x_init_value=x_init) print("leaky_relu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4)
def testGradGradFloat32(self): with compat.forward_compatibility_horizon(2018, 11, 2): with self.test_session(): x = constant_op.constant( [-0.9, -0.7, -0.5, -0.3, -0.1, 0.1, 0.3, 0.5, 0.7, 0.9], shape=[2, 5], name="x") y = nn_ops.leaky_relu(x, alpha=0.1, name="leaky_relu") z = gradients_impl.gradients(y, x) x_init = np.asarray( [[-0.9, -0.7, -0.5, -0.3, -0.1], [0.1, 0.3, 0.5, 0.7, 0.9]], dtype=np.float32, order="F") err = gradient_checker.compute_gradient_error( x, [2, 5], z[0], [2, 5], x_init_value=x_init) print("leaky_relu (float32) gradient of gradient err = ", err) self.assertLess(err, 1e-4)
def testValues(self): np_values = np.array([-1.0, 0.0, 0.5, 1.0, 2.0], dtype=np.float32) outputs = nn_ops.leaky_relu(constant_op.constant(np_values)) with self.test_session() as sess: outputs = sess.run(outputs) self.assertAllClose(outputs, [-0.2, 0.0, 0.5, 1.0, 2.0])
def __call__(self, text_input, training=False, dropout_rate=0.0): """ Runs the CNN producing the predictions and the gradients. :param text_input: Text input to produce embeddings for. e.g. for text data [batch_size, 300] :param training: A flag indicating training or evaluation :param dropout_rate: A tf placeholder of type tf.float32 indicating the amount of dropout applied :return: Embeddings of size [batch_size, self.num_classes] """ with tf.variable_scope(self.name, reuse=self.reuse): layer_features = [] with tf.variable_scope('VGGNet'): outputs = image_input for i in range(len(self.layer_stage_sizes)): with tf.variable_scope('conv_stage_{}'.format(i)): for j in range(self.inner_layer_depth): with tf.variable_scope('conv_{}_{}'.format(i, j)): if (j == self.inner_layer_depth - 1) and self.strided_dim_reduction: stride = 2 else: stride = 1 outputs = tf.layers.conv2d( outputs, self.layer_stage_sizes[i], [3, 3], strides=(stride, stride), padding='SAME', activation=None) outputs = leaky_relu( outputs, name="leaky_relu{}".format(i)) layer_features.append(outputs) if self.batch_norm_use: outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=False) if self.strided_dim_reduction == False: outputs = tf.layers.max_pooling2d(outputs, pool_size=(2, 2), strides=2) outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) # apply dropout only at dimensionality # reducing steps, i.e. the last layer in # every group c_conv_encoder = outputs c_conv_encoder = tf.contrib.layers.flatten(c_conv_encoder) c_conv_encoder = tf.layers.dense(c_conv_encoder, units=self.num_classes) self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) if not self.build_completed: self.build_completed = True count_parameters(self.variables, "VGGNet") return c_conv_encoder, layer_features
def _testLeakyRelu(self, np_features, alpha): np_leaky_relu = self._npLeakyRelu(np_features, alpha) tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha) self.assertAllClose(np_leaky_relu, tf_leaky_relu) self.assertShapeEqual(np_leaky_relu, tf_leaky_relu)
def f(x): assert x.dtype == dtypes.float64 with backprop.GradientTape() as tape: tape.watch(x) y = nn_ops.leaky_relu(x) return tape.gradient(y, x)
def loss(): return nn_ops.leaky_relu(x, 0.05)**2
def __call__(self, conditional_input, generated_input, training=False, dropout_rate=0.0): """ :param conditional_input: A batch of conditional inputs (x_i) of size [batch_size, height, width, channel] :param generated_input: A batch of generated inputs (x_g) of size [batch_size, height, width, channel] :param training: Placeholder for training or a boolean indicating training or validation :param dropout_rate: A float placeholder for dropout rate or a float indicating the dropout rate :param name: Network name :return: """ conditional_input = tf.convert_to_tensor(conditional_input) generated_input = tf.convert_to_tensor(generated_input) with tf.variable_scope(self.name, reuse=self.reuse): concat_images = tf.concat([conditional_input, generated_input], axis=3) outputs = concat_images encoder_layers = [] current_layers = [outputs] with tf.variable_scope('conv_layers'): for i, layer_size in enumerate(self.layer_sizes): encoder_inner_layers = [outputs] with tf.variable_scope('g_conv{}'.format(i)): if i == 0: outputs = self.conv_layer(outputs, num_filters=64, filter_size=(3, 3), strides=(2, 2)) outputs = leaky_relu(features=outputs) outputs = layer_norm(inputs=outputs, center=True, scale=True) current_layers.append(outputs) else: for j in range(self.inner_layers[i]): outputs = self.add_encoder_layer( input=outputs, name="encoder_inner_conv_{}_{}".format( i, j), training=training, layer_to_skip_connect=current_layers[-2], num_features=self.layer_sizes[i], dropout_rate=dropout_rate, dim_reduce=False, local_inner_layers=encoder_inner_layers) current_layers.append(outputs) outputs = self.add_encoder_layer( input=outputs, name="encoder_outer_conv_{}".format(i), training=training, layer_to_skip_connect=current_layers[-2], local_inner_layers=encoder_inner_layers, num_features=self.layer_sizes[i], dropout_rate=dropout_rate, dim_reduce=True) current_layers.append(outputs) encoder_layers.append(outputs) flatten = tf.contrib.layers.flatten(encoder_layers[-1]) with tf.variable_scope('discriminator_out'): outputs = tf.layers.dense(flatten, 1, name='outputs') self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) #view_names_of_variables(self.variables) if self.build: print("discr layers", self.conv_layer_num) count_parameters(self.variables, name="discriminator_parameter_num") self.build = False return outputs, current_layers
def __call__(self, inputs, input_y, classes, training=False, dropout_rate=0.0): inputs = tf.convert_to_tensor(inputs) with tf.variable_scope(self.name, reuse=self.reuse): outputs = inputs encoder_layers = [] current_layers = [outputs] with tf.variable_scope('conv_layers'): for i, layer_size in enumerate(self.layer_sizes): encoder_inner_layers = [outputs] with tf.variable_scope('g_conv{}'.format(i)): if i == 0: outputs = self.conv_layer(outputs, num_filters=64, filter_size=(3, 3), strides=(2, 2)) outputs = leaky_relu(features=outputs) outputs = layer_norm(inputs=outputs, center=True, scale=True) # outputs = tf.nn.relu(outputs) # outputs = tf.layers.batch_normalization(outputs, training=training, momentum=0.9) current_layers.append(outputs) else: for j in range(self.inner_layers[i]): outputs = self.add_encoder_layer( input=outputs, name="encoder_inner_conv_{}_{}".format( i, j), training=training, layer_to_skip_connect=current_layers[-2], num_features=self.layer_sizes[i], dropout_rate=dropout_rate, dim_reduce=False, local_inner_layers=encoder_inner_layers) current_layers.append(outputs) encoder_inner_layers.append(outputs) outputs = self.add_encoder_layer( input=outputs, name="encoder_outer_conv_{}".format(i), training=training, layer_to_skip_connect=current_layers[-2], local_inner_layers=encoder_inner_layers, num_features=self.layer_sizes[i], dropout_rate=dropout_rate, dim_reduce=True) current_layers.append(outputs) encoder_layers.append(outputs) with tf.variable_scope('classifier_dense_block'): if self.use_wide_connections: mean_encoder_layers = [] concat_encoder_layers = [] for layer in encoder_layers: mean_encoder_layers.append( tf.reduce_mean(layer, axis=[1, 2])) concat_encoder_layers.append(tf.layers.flatten(layer)) feature_level_flatten = tf.concat(mean_encoder_layers, axis=1) location_level_flatten = tf.concat(concat_encoder_layers, axis=1) else: feature_level_flatten = tf.reduce_mean(encoder_layers[-1], axis=[1, 2]) location_level_flatten = tf.layers.flatten( encoder_layers[-1]) feature_level_dense = tf.layers.dense(feature_level_flatten, units=1024, activation=leaky_relu) # feature_level_dense = tf.layers.dense(feature_level_flatten, units=1024) combo_level_flatten = tf.concat( [feature_level_dense, location_level_flatten], axis=1) with tf.variable_scope('classifier_out_block'): logits = tf.layers.dense(combo_level_flatten, units=classes) cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=input_y, logits=logits)) correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(input_y, 1)) accuracy = tf.reduce_mean( tf.cast(correct_prediction, tf.float32)) # cost = tf.losses.softmax_cross_entropy(onehot_labels=input_y, logits=logits) # correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(input_y, 1)) # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # cost = tf.losses.softmax_cross_entropy(onehot_labels=input_y, logits=logits) # correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(input_y, 1)) # # correct_prediction=tf.Print(correct_prediction,[correct_prediction],'correct_prediction') # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.name) if self.build: print("classification layers", self.conv_layer_num) count_parameters(self.variables, name="classifier_parameter_num") self.build = False return cost, accuracy
def f(x): assert x.dtype == dtypes.float64 with backprop.GradientTape() as tape: tape.watch(x) y = nn_ops.leaky_relu(x) return tape.gradient(y, x)
def _testLeakyRelu(self, np_features, alpha): np_leaky_relu = self._npLeakyRelu(np_features, alpha) tf_leaky_relu = nn_ops.leaky_relu(np_features, alpha) self.assertAllClose(np_leaky_relu, tf_leaky_relu) self.assertShapeEqual(np_leaky_relu, tf_leaky_relu)
def add_decoder_layer(self, input, name, training, dropout_rate, layer_to_skip_connect, local_inner_layers, num_features, dim_upscale=False, h_size=None, w_size=None): """ Adds a resnet decoder layer. :param input: Input features :param name: Layer Name :param training: Training placeholder or boolean flag :param dropout_rate: Float placeholder or float indicating the dropout rate :param layer_to_skip_connect: Layer to skip connect to. :param local_inner_layers: A list with the inner layers of the current MultiLayer :param num_features: Num feature maps for convolution :param dim_upscale: Dimensionality upscale :param h_size: Height to upscale to :param w_size: Width to upscale to :return: The output of the decoder layer """ [b1, h1, w1, d1] = input.get_shape().as_list() if len(layer_to_skip_connect) >= 2: layer_to_skip_connect = layer_to_skip_connect[-2] else: layer_to_skip_connect = None if layer_to_skip_connect is not None: [b0, h0, w0, d0] = layer_to_skip_connect.get_shape().as_list() if h0 < h1: skip_connect_layer = self.conv_layer( layer_to_skip_connect, int(layer_to_skip_connect.get_shape()[3]), [1, 1], strides=(1, 1), transpose=True, h_size=h_size, w_size=w_size) else: skip_connect_layer = layer_to_skip_connect current_layers = [input, skip_connect_layer] else: current_layers = [input] current_layers.extend(local_inner_layers) current_layers = remove_duplicates(current_layers) outputs = tf.concat(current_layers, axis=3) if dim_upscale: outputs = self.conv_layer(outputs, num_features, [3, 3], strides=(1, 1), transpose=True, w_size=w_size, h_size=h_size) outputs = leaky_relu(features=outputs) outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=True) outputs = tf.layers.dropout(outputs, rate=dropout_rate, training=training) else: outputs = self.conv_layer(outputs, num_features, [3, 3], strides=(1, 1), transpose=False) outputs = leaky_relu(features=outputs) outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=True) return outputs
def loss(): return nn_ops.leaky_relu(x, 0.05)**2
def __call__(self, z_inputs, conditional_input, training=False, dropout_rate=0.0): """ Apply network on data. :param z_inputs: Random noise to inject [batch_size, z_dim] :param conditional_input: A batch of images to use as conditionals [batch_size, height, width, channels] :param training: Training placeholder or boolean :param dropout_rate: Dropout rate placeholder or float :return: Returns x_g (generated images), encoder_layers(encoder features), decoder_layers(decoder features) """ conditional_input = tf.convert_to_tensor(conditional_input) with tf.variable_scope(self.name, reuse=self.reuse): # reshape from inputs outputs = conditional_input encoder_layers = [] current_layers = [outputs] with tf.variable_scope('conv_layers'): for i, layer_size in enumerate(self.layer_sizes): encoder_inner_layers = [outputs] with tf.variable_scope('g_conv{}'.format(i)): if i == 0: #first layer is a single conv layer instead of MultiLayer for best results outputs = self.conv_layer(outputs, num_filters=64, filter_size=(3, 3), strides=(2, 2)) outputs = leaky_relu(features=outputs) outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=True) current_layers.append(outputs) encoder_inner_layers.append(outputs) else: for j in range( self.inner_layers[i] ): #Build the inner Layers of the MultiLayer outputs = self.add_encoder_layer( input=outputs, training=training, name="encoder_layer_{}_{}".format(i, j), layer_to_skip_connect=current_layers, num_features=self.layer_sizes[i], dim_reduce=False, local_inner_layers=encoder_inner_layers, dropout_rate=dropout_rate) encoder_inner_layers.append(outputs) current_layers.append(outputs) #add final dim reducing conv layer for this MultiLayer outputs = self.add_encoder_layer( input=outputs, name="encoder_layer_{}".format(i), training=training, layer_to_skip_connect=current_layers, local_inner_layers=encoder_inner_layers, num_features=self.layer_sizes[i], dim_reduce=True, dropout_rate=dropout_rate) current_layers.append(outputs) encoder_layers.append(outputs) g_conv_encoder = outputs with tf.variable_scope( "vector_expansion" ): # Used for expanding the z injected noise to match the # dimensionality of the various decoder MultiLayers, injecting # noise into multiple decoder layers in a skip-connection way # improves quality of results. We inject in the first 3 decode # multi layers num_filters = 8 concat_shape = tuple(encoder_layers[-1].get_shape()) concat_shape = [int(i) for i in concat_shape] z_dense_0 = tf.layers.dense(z_inputs, concat_shape[1] * concat_shape[2] * num_filters, name='input_dense_0') z_reshape_0 = tf.reshape(z_dense_0, [ self.batch_size, concat_shape[1], concat_shape[2], num_filters ], name='z_reshape_0') concat_shape = tuple(encoder_layers[-2].get_shape()) concat_shape = [int(i) for i in concat_shape] z_dense_1 = tf.layers.dense(z_inputs, concat_shape[1] * concat_shape[2] * num_filters, name='input_dense_1') z_reshape_1 = tf.reshape(z_dense_1, [ self.batch_size, concat_shape[1], concat_shape[2], num_filters ], name='z_reshape_1') num_filters = num_filters / 2 concat_shape = tuple(encoder_layers[-3].get_shape()) concat_shape = [int(i) for i in concat_shape] z_dense_2 = tf.layers.dense(z_inputs, concat_shape[1] * concat_shape[2] * num_filters, name='input_dense_2') z_reshape_2 = tf.reshape(z_dense_2, [ self.batch_size, concat_shape[1], concat_shape[2], num_filters ], name='z_reshape_2') num_filters = num_filters / 2 concat_shape = tuple(encoder_layers[-4].get_shape()) concat_shape = [int(i) for i in concat_shape] z_dense_3 = tf.layers.dense(z_inputs, concat_shape[1] * concat_shape[2] * num_filters, name='input_dense_3') z_reshape_3 = tf.reshape(z_dense_3, [ self.batch_size, concat_shape[1], concat_shape[2], num_filters ], name='z_reshape_3') z_layers = [z_reshape_0, z_reshape_1, z_reshape_2, z_reshape_3] outputs = g_conv_encoder decoder_layers = [] current_layers = [outputs] with tf.variable_scope('g_deconv_layers'): for i in range(len(self.layer_sizes) + 1): if i < 3: #Pass the injected noise to the first 3 decoder layers for sharper results outputs = tf.concat([z_layers[i], outputs], axis=3) current_layers[-1] = outputs idx = len(self.layer_sizes) - 1 - i num_features = self.layer_sizes[idx] inner_layers = self.inner_layers[idx] upscale_shape = encoder_layers[idx].get_shape().as_list() if idx < 0: num_features = self.layer_sizes[0] inner_layers = self.inner_layers[0] outputs = tf.concat([outputs, conditional_input], axis=3) upscale_shape = conditional_input.get_shape().as_list() with tf.variable_scope('g_deconv{}'.format(i)): decoder_inner_layers = [outputs] for j in range(inner_layers): if i == 0 and j == 0: outputs = self.add_decoder_layer( input=outputs, name="decoder_inner_conv_{}_{}".format( i, j), training=training, layer_to_skip_connect=current_layers, num_features=num_features, dim_upscale=False, local_inner_layers=decoder_inner_layers, dropout_rate=dropout_rate) decoder_inner_layers.append(outputs) else: outputs = self.add_decoder_layer( input=outputs, name="decoder_inner_conv_{}_{}".format( i, j), training=training, layer_to_skip_connect=current_layers, num_features=num_features, dim_upscale=False, local_inner_layers=decoder_inner_layers, w_size=upscale_shape[1], h_size=upscale_shape[2], dropout_rate=dropout_rate) decoder_inner_layers.append(outputs) current_layers.append(outputs) decoder_layers.append(outputs) if idx >= 0: upscale_shape = encoder_layers[ idx - 1].get_shape().as_list() if idx == 0: upscale_shape = conditional_input.get_shape( ).as_list() outputs = self.add_decoder_layer( input=outputs, name="decoder_outer_conv_{}".format(i), training=training, layer_to_skip_connect=current_layers, num_features=num_features, dim_upscale=True, local_inner_layers=decoder_inner_layers, w_size=upscale_shape[1], h_size=upscale_shape[2], dropout_rate=dropout_rate) current_layers.append(outputs) if (idx - 1) >= 0: outputs = tf.concat( [outputs, encoder_layers[idx - 1]], axis=3) current_layers[-1] = outputs high_res_layers = [] for p in range(2): outputs = self.conv_layer(outputs, self.layer_sizes[0], [3, 3], strides=(1, 1), transpose=False) outputs = leaky_relu(features=outputs) outputs = batch_norm(outputs, decay=0.99, scale=True, center=True, is_training=training, renorm=True) high_res_layers.append(outputs) outputs = self.conv_layer(outputs, self.num_channels, [3, 3], strides=(1, 1), transpose=False) # output images with tf.variable_scope('g_tanh'): gan_decoder = tf.tanh(outputs, name='outputs') self.reuse = True self.variables = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='g') if self.build: print("generator_total_layers", self.conv_layer_num) count_parameters(self.variables, name="generator_parameter_num") self.build = False return gan_decoder, encoder_layers, decoder_layers