def deepnn(x, n_classes): is_train = tf.placeholder(tf.bool) keep_prob = tf.placeholder(tf.float32) h = tf.contrib.layers.fully_connected( x, 256, activation_fn=None, weights_regularizer=tf.contrib.layers.l1_regularizer(scale=0.0005)) # h = tf.keras.layers.Dense(1000)(x) h = taylor(h, k=2, is_train=is_train, name="Ac/1") # h = tf.layers.batch_normalization(h, training=is_train) # h = tf.nn.tanh(h) h = tf.contrib.layers.fully_connected(h, 128, activation_fn=None) # h = tf.keras.layers.Dense(1000)(x) h = taylor(h, k=2, is_train=is_train, name="Ac/2") # h = tf.layers.batch_normalization(h, training=is_train) # h = tf.nn.tanh(h) h = tf.contrib.layers.fully_connected(h, 64, activation_fn=None) # h = tf.keras.layers.Dense(1000)(x) h = taylor(h, k=2, is_train=is_train, name="Ac/3") # h = tf.contrib.layers.fully_connected(h, 2048, activation_fn=None) # h = taylor(h, k=2, is_train=is_train, name="Ac/2") # h = tf.keras.layers.Dense(1)(h) h = tf.contrib.layers.fully_connected(h, n_classes, activation_fn=None) return h, is_train, keep_prob
def hidden_RAMO(is_train): h1 = tf.contrib.layers.fully_connected(x, 128, activation_fn=None, weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.05)) h1 = taylor(h1, k=4, is_train=is_train, name='Ac/1') h1 = tf.contrib.layers.fully_connected(h1, 64, activation_fn=None, weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.05)) h1 = taylor(h1, k=2, is_train=is_train, name='Ac/3') h1 = tf.keras.layers.Dense(1)(h1) # t = activation_function_taylor(t, k=4, is_train=is_train) return h1
def hidden_RAMO(is_train): n_classes = 1 dim = 2 with tf.variable_scope("model"): val = 20 W1 = create_weight_variable('Weights', [dim, val]) b1 = create_bias_variable('Bias', [val]) W2 = create_weight_variable('Weights2', [val, val]) b2 = create_bias_variable('Bias2', [val]) # W3 = create_weight_variable('Weights3', [5, 5]) # b3 = create_bias_variable('Bias3', [5]) W4 = create_weight_variable('Weights4', [val, n_classes]) b4 = create_bias_variable('Bias4', [n_classes]) h1 = taylor(tf.matmul(x, W1) + b1, k=7, is_train=is_train, name='Ac/1') # h1 = tf.nn.relu(tf.matmul(x, W1) + b1) h3 = taylor(tf.matmul(h1, W2) + b2, k=7, is_train=is_train, name='Ac/2') t = (tf.matmul(h3, W4) + b4) # t = activation_function_taylor(t, k=4, is_train=is_train) return t, t
def __call__(self, inputs, training): """Add operations to classify a batch of input images. Args: inputs: A Tensor representing a batch of input images. training: A boolean. Set to True to add operations required only when training the classifier. Returns: A logits Tensor with shape [<batch_size>, self.num_classes]. """ with self._model_variable_scope(): if self.data_format == 'channels_first': # Convert the inputs from channels_last (NHWC) to channels_first (NCHW). # This provides a large performance boost on GPU. See # https://www.tensorflow.org/performance/performance_guide#data_formats inputs = tf.transpose(inputs, [0, 3, 1, 2]) inputs = conv2d_fixed_padding(inputs=inputs, filters=self.num_filters, kernel_size=self.kernel_size, strides=self.conv_stride, data_format=self.data_format) inputs = tf.identity(inputs, 'initial_conv') # We do not include batch normalization or activation functions in V2 # for the initial conv1 because the first ResNet unit will perform these # for both the shortcut and non-shortcut paths as part of the first # block's projection. Cf. Appendix of [2]. # THis line was commented # if self.resnet_version == 1: # inputs = batch_norm(inputs, training, self.data_format) # inputs = tf.nn.relu(inputs) if self.resnet_version == 1: inputs = taylor(inputs, k=4, name="activation/ac1", is_train=training) with self._model_variable_scope(name='resnet_model_1'): if self.first_pool_size: inputs = tf.layers.max_pooling2d( inputs=inputs, pool_size=self.first_pool_size, strides=self.first_pool_stride, padding='SAME', data_format=self.data_format) inputs = tf.identity(inputs, 'initial_max_pool') for i, num_blocks in enumerate(self.block_sizes): num_filters = self.num_filters * (2**i) if i < 1: inputs = block_layer(inputs=inputs, filters=num_filters, bottleneck=self.bottleneck, block_fn=self.block_fn, blocks=num_blocks, strides=self.block_strides[i], training=training, name='block_layer{}'.format(i + 1), data_format=self.data_format, replace=True, ac_name='activation' + str(i) + '_' + str(num_blocks) + '_ac') else: inputs = block_layer(inputs=inputs, filters=num_filters, bottleneck=self.bottleneck, block_fn=self.block_fn, blocks=num_blocks, strides=self.block_strides[i], training=training, name='block_layer{}'.format(i + 1), data_format=self.data_format, replace=True, ac_name='activation' + str(i) + '_' + str(num_blocks) + '_ac') # Only apply the BN and ReLU for model that does pre_activation in each # building/bottleneck block, eg resnet V2. if self.pre_activation: # inputs = batch_norm(inputs, training, self.data_format) # inputs = tf.nn.relu(inputs) inputs = taylor(inputs, k=4, name="activation_pre_activation/ac1", is_train=training) # The current top layer has shape # `batch_size x pool_size x pool_size x final_size`. # ResNet does an Average Pooling layer over pool_size, # but that is the same as doing a reduce_mean. We do a reduce_mean # here because it performs better than AveragePooling2D. axes = [2, 3] if self.data_format == 'channels_first' else [1, 2] inputs = tf.reduce_mean(inputs, axes, keepdims=True) inputs = tf.identity(inputs, 'final_reduce_mean') inputs = tf.squeeze(inputs, axes) # inputs = tf.layers.dense(inputs=inputs, units=1000) # inputs = taylor(inputs, k=3, name="activation_1") with self._model_variable_scope(name='resnet_model_2'): inputs = tf.layers.dense(inputs=inputs, units=self.num_classes) inputs = tf.identity(inputs, 'final_dense') return inputs
def _building_block_v1(inputs, filters, training, projection_shortcut, strides, data_format, replace=False, ac_name1='taylor_activation'): """A single block for ResNet v1, without a bottleneck. Convolution then batch normalization then ReLU as described by: Deep Residual Learning for Image Recognition https://arxiv.org/pdf/1512.03385.pdf by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015. Args: inputs: A tensor of size [batch, channels, height_in, width_in] or [batch, height_in, width_in, channels] depending on data_format. filters: The number of filters for the convolutions. training: A Boolean for whether the model is in training or inference mode. Needed for batch normalization. projection_shortcut: The function to use for projection shortcuts (typically a 1x1 convolution when downsampling the input). strides: The block's stride. If greater than 1, this block will ultimately downsample the input. data_format: The input format ('channels_last' or 'channels_first'). Returns: The output tensor of the block; shape should match inputs. """ shortcut = inputs if projection_shortcut is not None: shortcut = projection_shortcut(inputs) # shortcut = batch_norm(inputs=shortcut, training=training, # data_format=data_format) inputs = conv2d_fixed_padding(inputs=inputs, filters=filters, kernel_size=3, strides=strides, data_format=data_format) if replace: inputs = taylor(inputs, k=2, name='activation/ac2/' + ac_name1, is_train=training) else: inputs = batch_norm(inputs, training, data_format) inputs = tf.nn.relu(inputs) inputs = conv2d_fixed_padding(inputs=inputs, filters=filters, kernel_size=2, strides=1, data_format=data_format) # inputs += shortcut if replace: inputs += shortcut inputs = taylor(inputs, k=2, name='activation/ac2/' + ac_name1 + '2', is_train=training) else: inputs = batch_norm(inputs, training, data_format) inputs += shortcut inputs = tf.nn.relu(inputs) return inputs
def deepnn(x, is_train): with tf.name_scope('reshape'): x_image = tf.reshape(x, [-1, 28, 28, 1]) # First convolutional layer - maps one grayscale image to 32 feature maps. with tf.name_scope('conv1'): W_conv1 = weight_variable([5, 5, 1, 16]) b_conv1 = bias_variable([16]) temp_1 = b_conv1 #a = tf.reshape(tf.size(x_image), [-1, 1]) temp_1 = (conv2d(x_image, W_conv1) + b_conv1) #UC # _, _,out,_, _ = non_linear_Ac(temp_1, input_size = 25088, k1 = 4, k2 = 4, output_size = 25088) #UC # h_conv1, w1, b1 = RAMO(temp_1, k1=4, k2=4, scale=None) #UC h_conv1 = taylor(temp_1, k=4, is_train=is_train, name="Ac/1") # h_conv1 = tf.nn.relu(temp_1) print(h_conv1, "RAJANNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN") # h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) #C # Pooling layer - downsamples by 2X. with tf.name_scope('pool1'): h_pool1 = max_pool_2x2(h_conv1) # Second convolutional layer -- maps 16 feature maps to 64. with tf.name_scope('conv2'): W_conv2 = weight_variable([5, 5, 16, 32]) b_conv2 = bias_variable([32]) temp_c2 = (conv2d(h_pool1, W_conv2) + b_conv2) #UC # h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) #C # _,_,out_c2,_, _ = non_linear_Ac(temp_c2, input_size = 14*14*32, k1 = 4, k2 = 4, output_size = 14*14*32) #UC # h_conv2 = tf.reshape(out_c2, [-1, 14, 14, 32]) #UC h_conv2 = taylor(temp_c2, k=4, is_train=is_train, name="Ac/2") # h_conv2 = tf.nn.relu(temp_c2) # Second pooling layer. with tf.name_scope('pool2'): h_pool2 = max_pool_2x2(h_conv2) # Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image # is down to 7x7x32 feature maps -- maps this to 1024 features. with tf.name_scope('fc1'): W_fc1 = weight_variable([7 * 7 * 32, 1024]) b_fc1 = bias_variable([1024]) h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*32]) # h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) #C h_fc = tf.matmul(h_pool2_flat, W_fc1) + b_fc1 #UC # w, b, h_fc1, _, m = non_linear_Ac(h_fc, input_size=1024, k1=8, k2=8, scale=None, output_size=1024) #UC h_fc1 = taylor(h_fc, k=5, is_train=is_train, name="Ac/3") # h_fc1 = tf.nn.relu(h_fc) # Dropout - controls the complexity of the model, prevents co-adaptation of # features. with tf.name_scope('dropout'): keep_prob = tf.placeholder(tf.float32) h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob) # Map the 1024 features to 10 classes, one for each digit with tf.name_scope('fc2'): W_fc2 = weight_variable([1024, 10]) b_fc2 = bias_variable([10]) y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2 return y_conv, keep_prob