def deepnn(x, n_classes):
is_train = tf.placeholder(tf.bool)
keep_prob = tf.placeholder(tf.float32)
h = tf.contrib.layers.fully_connected(
x,
256,
activation_fn=None,
weights_regularizer=tf.contrib.layers.l1_regularizer(scale=0.0005))
# h = tf.keras.layers.Dense(1000)(x)
h = taylor(h, k=2, is_train=is_train, name="Ac/1")
# h = tf.layers.batch_normalization(h, training=is_train)
# h = tf.nn.tanh(h)
h = tf.contrib.layers.fully_connected(h, 128, activation_fn=None)
# h = tf.keras.layers.Dense(1000)(x)
h = taylor(h, k=2, is_train=is_train, name="Ac/2")
# h = tf.layers.batch_normalization(h, training=is_train)
# h = tf.nn.tanh(h)
h = tf.contrib.layers.fully_connected(h, 64, activation_fn=None)
# h = tf.keras.layers.Dense(1000)(x)
h = taylor(h, k=2, is_train=is_train, name="Ac/3")
# h = tf.contrib.layers.fully_connected(h, 2048, activation_fn=None)
# h = taylor(h, k=2, is_train=is_train, name="Ac/2")
# h = tf.keras.layers.Dense(1)(h)
h = tf.contrib.layers.fully_connected(h, n_classes, activation_fn=None)
return h, is_train, keep_prob
def hidden_RAMO(is_train):
h1 = tf.contrib.layers.fully_connected(x, 128, activation_fn=None, weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.05))
h1 = taylor(h1, k=4, is_train=is_train, name='Ac/1')
h1 = tf.contrib.layers.fully_connected(h1, 64, activation_fn=None, weights_regularizer=tf.contrib.layers.l2_regularizer(scale=0.05))
h1 = taylor(h1, k=2, is_train=is_train, name='Ac/3')
h1 = tf.keras.layers.Dense(1)(h1)
# t = activation_function_taylor(t, k=4, is_train=is_train)
return h1
def hidden_RAMO(is_train):
n_classes = 1
dim = 2
with tf.variable_scope("model"):
val = 20
W1 = create_weight_variable('Weights', [dim, val])
b1 = create_bias_variable('Bias', [val])
W2 = create_weight_variable('Weights2', [val, val])
b2 = create_bias_variable('Bias2', [val])
# W3 = create_weight_variable('Weights3', [5, 5])
# b3 = create_bias_variable('Bias3', [5])
W4 = create_weight_variable('Weights4', [val, n_classes])
b4 = create_bias_variable('Bias4', [n_classes])
h1 = taylor(tf.matmul(x, W1) + b1, k=7, is_train=is_train, name='Ac/1')
# h1 = tf.nn.relu(tf.matmul(x, W1) + b1)
h3 = taylor(tf.matmul(h1, W2) + b2, k=7, is_train=is_train, name='Ac/2')
t = (tf.matmul(h3, W4) + b4)
# t = activation_function_taylor(t, k=4, is_train=is_train)
return t, t
def __call__(self, inputs, training):
"""Add operations to classify a batch of input images.
Args:
inputs: A Tensor representing a batch of input images.
training: A boolean. Set to True to add operations required only when
training the classifier.
Returns:
A logits Tensor with shape [<batch_size>, self.num_classes].
"""
with self._model_variable_scope():
if self.data_format == 'channels_first':
# Convert the inputs from channels_last (NHWC) to channels_first (NCHW).
# This provides a large performance boost on GPU. See
# https://www.tensorflow.org/performance/performance_guide#data_formats
inputs = tf.transpose(inputs, [0, 3, 1, 2])
inputs = conv2d_fixed_padding(inputs=inputs,
filters=self.num_filters,
kernel_size=self.kernel_size,
strides=self.conv_stride,
data_format=self.data_format)
inputs = tf.identity(inputs, 'initial_conv')
# We do not include batch normalization or activation functions in V2
# for the initial conv1 because the first ResNet unit will perform these
# for both the shortcut and non-shortcut paths as part of the first
# block's projection. Cf. Appendix of [2].
# THis line was commented
# if self.resnet_version == 1:
# inputs = batch_norm(inputs, training, self.data_format)
# inputs = tf.nn.relu(inputs)
if self.resnet_version == 1:
inputs = taylor(inputs,
k=4,
name="activation/ac1",
is_train=training)
with self._model_variable_scope(name='resnet_model_1'):
if self.first_pool_size:
inputs = tf.layers.max_pooling2d(
inputs=inputs,
pool_size=self.first_pool_size,
strides=self.first_pool_stride,
padding='SAME',
data_format=self.data_format)
inputs = tf.identity(inputs, 'initial_max_pool')
for i, num_blocks in enumerate(self.block_sizes):
num_filters = self.num_filters * (2**i)
if i < 1:
inputs = block_layer(inputs=inputs,
filters=num_filters,
bottleneck=self.bottleneck,
block_fn=self.block_fn,
blocks=num_blocks,
strides=self.block_strides[i],
training=training,
name='block_layer{}'.format(i + 1),
data_format=self.data_format,
replace=True,
ac_name='activation' + str(i) + '_' +
str(num_blocks) + '_ac')
else:
inputs = block_layer(inputs=inputs,
filters=num_filters,
bottleneck=self.bottleneck,
block_fn=self.block_fn,
blocks=num_blocks,
strides=self.block_strides[i],
training=training,
name='block_layer{}'.format(i + 1),
data_format=self.data_format,
replace=True,
ac_name='activation' + str(i) + '_' +
str(num_blocks) + '_ac')
# Only apply the BN and ReLU for model that does pre_activation in each
# building/bottleneck block, eg resnet V2.
if self.pre_activation:
# inputs = batch_norm(inputs, training, self.data_format)
# inputs = tf.nn.relu(inputs)
inputs = taylor(inputs,
k=4,
name="activation_pre_activation/ac1",
is_train=training)
# The current top layer has shape
# `batch_size x pool_size x pool_size x final_size`.
# ResNet does an Average Pooling layer over pool_size,
# but that is the same as doing a reduce_mean. We do a reduce_mean
# here because it performs better than AveragePooling2D.
axes = [2, 3] if self.data_format == 'channels_first' else [1, 2]
inputs = tf.reduce_mean(inputs, axes, keepdims=True)
inputs = tf.identity(inputs, 'final_reduce_mean')
inputs = tf.squeeze(inputs, axes)
# inputs = tf.layers.dense(inputs=inputs, units=1000)
# inputs = taylor(inputs, k=3, name="activation_1")
with self._model_variable_scope(name='resnet_model_2'):
inputs = tf.layers.dense(inputs=inputs, units=self.num_classes)
inputs = tf.identity(inputs, 'final_dense')
return inputs
def _building_block_v1(inputs,
filters,
training,
projection_shortcut,
strides,
data_format,
replace=False,
ac_name1='taylor_activation'):
"""A single block for ResNet v1, without a bottleneck.
Convolution then batch normalization then ReLU as described by:
Deep Residual Learning for Image Recognition
https://arxiv.org/pdf/1512.03385.pdf
by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Dec 2015.
Args:
inputs: A tensor of size [batch, channels, height_in, width_in] or
[batch, height_in, width_in, channels] depending on data_format.
filters: The number of filters for the convolutions.
training: A Boolean for whether the model is in training or inference
mode. Needed for batch normalization.
projection_shortcut: The function to use for projection shortcuts
(typically a 1x1 convolution when downsampling the input).
strides: The block's stride. If greater than 1, this block will ultimately
downsample the input.
data_format: The input format ('channels_last' or 'channels_first').
Returns:
The output tensor of the block; shape should match inputs.
"""
shortcut = inputs
if projection_shortcut is not None:
shortcut = projection_shortcut(inputs)
# shortcut = batch_norm(inputs=shortcut, training=training,
# data_format=data_format)
inputs = conv2d_fixed_padding(inputs=inputs,
filters=filters,
kernel_size=3,
strides=strides,
data_format=data_format)
if replace:
inputs = taylor(inputs,
k=2,
name='activation/ac2/' + ac_name1,
is_train=training)
else:
inputs = batch_norm(inputs, training, data_format)
inputs = tf.nn.relu(inputs)
inputs = conv2d_fixed_padding(inputs=inputs,
filters=filters,
kernel_size=2,
strides=1,
data_format=data_format)
# inputs += shortcut
if replace:
inputs += shortcut
inputs = taylor(inputs,
k=2,
name='activation/ac2/' + ac_name1 + '2',
is_train=training)
else:
inputs = batch_norm(inputs, training, data_format)
inputs += shortcut
inputs = tf.nn.relu(inputs)
return inputs
def deepnn(x, is_train):
with tf.name_scope('reshape'):
x_image = tf.reshape(x, [-1, 28, 28, 1])
# First convolutional layer - maps one grayscale image to 32 feature maps.
with tf.name_scope('conv1'):
W_conv1 = weight_variable([5, 5, 1, 16])
b_conv1 = bias_variable([16])
temp_1 = b_conv1
#a = tf.reshape(tf.size(x_image), [-1, 1])
temp_1 = (conv2d(x_image, W_conv1) + b_conv1) #UC
# _, _,out,_, _ = non_linear_Ac(temp_1, input_size = 25088, k1 = 4, k2 = 4, output_size = 25088) #UC
# h_conv1, w1, b1 = RAMO(temp_1, k1=4, k2=4, scale=None) #UC
h_conv1 = taylor(temp_1, k=4, is_train=is_train, name="Ac/1")
# h_conv1 = tf.nn.relu(temp_1)
print(h_conv1, "RAJANNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN")
# h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1) #C
# Pooling layer - downsamples by 2X.
with tf.name_scope('pool1'):
h_pool1 = max_pool_2x2(h_conv1)
# Second convolutional layer -- maps 16 feature maps to 64.
with tf.name_scope('conv2'):
W_conv2 = weight_variable([5, 5, 16, 32])
b_conv2 = bias_variable([32])
temp_c2 = (conv2d(h_pool1, W_conv2) + b_conv2) #UC
# h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2) #C
# _,_,out_c2,_, _ = non_linear_Ac(temp_c2, input_size = 14*14*32, k1 = 4, k2 = 4, output_size = 14*14*32) #UC
# h_conv2 = tf.reshape(out_c2, [-1, 14, 14, 32]) #UC
h_conv2 = taylor(temp_c2, k=4, is_train=is_train, name="Ac/2")
# h_conv2 = tf.nn.relu(temp_c2)
# Second pooling layer.
with tf.name_scope('pool2'):
h_pool2 = max_pool_2x2(h_conv2)
# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image
# is down to 7x7x32 feature maps -- maps this to 1024 features.
with tf.name_scope('fc1'):
W_fc1 = weight_variable([7 * 7 * 32, 1024])
b_fc1 = bias_variable([1024])
h_pool2_flat = tf.reshape(h_pool2, [-1, 7*7*32])
# h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1) #C
h_fc = tf.matmul(h_pool2_flat, W_fc1) + b_fc1 #UC
# w, b, h_fc1, _, m = non_linear_Ac(h_fc, input_size=1024, k1=8, k2=8, scale=None, output_size=1024) #UC
h_fc1 = taylor(h_fc, k=5, is_train=is_train, name="Ac/3")
# h_fc1 = tf.nn.relu(h_fc)
# Dropout - controls the complexity of the model, prevents co-adaptation of
# features.
with tf.name_scope('dropout'):
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
# Map the 1024 features to 10 classes, one for each digit
with tf.name_scope('fc2'):
W_fc2 = weight_variable([1024, 10])
b_fc2 = bias_variable([10])
y_conv = tf.matmul(h_fc1_drop, W_fc2) + b_fc2
return y_conv, keep_prob