def lenet300_100_model_fn(features, labels, mode):
input_layer = features['x']
with tf.variable_scope('dense1') as scope:
weights = _variable_with_weight_decay('weights',
shape=[784, 300],
stddev=0.04,
wd=0.004)
biases = _variable_on_cpu('biases', [300],
tf.constant_initializer(0.1))
pruned_weights = pruning.apply_mask(weights, scope)
_mask_summary(pruned_weights)
dense1 = tf.nn.relu(tf.matmul(input_layer, pruned_weights) +
biases,
name=scope.name)
_activation_summary(dense1)
with tf.variable_scope('dense2') as scope:
weights = _variable_with_weight_decay('weights',
shape=[300, 100],
stddev=0.04,
wd=0.004)
biases = _variable_on_cpu('biases', [100],
tf.constant_initializer(0.1))
pruned_weights = pruning.apply_mask(weights, scope)
_mask_summary(pruned_weights)
dense2 = tf.nn.relu(tf.matmul(dense1, pruned_weights) + biases,
name=scope.name)
_activation_summary(dense2)
with tf.variable_scope('logits') as scope:
weights = _variable_with_weight_decay('weights',
shape=[100, 10],
stddev=0.04,
wd=0.004)
biases = _variable_on_cpu('biases', [10],
tf.constant_initializer(0.1))
pruned_weights = pruning.apply_mask(weights, scope)
_mask_summary(pruned_weights)
logits = tf.nn.relu(tf.matmul(dense2, pruned_weights) + biases,
name=scope.name)
_activation_summary(logits)
return estimator_fn_complete(pruning_hparam_args, features, labels,
mode, logits)
def toy_numpy(features):
input_layer = features
with tf.variable_scope('logits') as scope:
W = _variable_with_weight_decay('W', [784, 10],
stddev=1 / 192.0,
wd=0.0)
pruned_weights = pruning.apply_mask(W, scope)
logits = tf.matmul(input_layer, W)
_activation_summary(logits)
return logits
def fc_layer(last_layer, shape, scope):
weights = _variable_with_weight_decay('weights',
shape=shape,
stddev=0.04,
wd=0.004)
biases = _variable_on_cpu('biases', shape[1], tf.constant_initializer(0.1))
pruned_weights = pruning.apply_mask(weights, scope)
_mask_summary(pruned_weights)
dense = tf.nn.relu(tf.matmul(last_layer, pruned_weights) + biases,
name=scope.name)
_activation_summary(dense)
return dense
def baby_net_model_fn(features, labels, mode):
layer1_size = 50
# Input Layer
input_layer = features['x']
input_size = np.prod(input_layer.shape.as_list()[1:])
# fully connected layer 1
with tf.variable_scope('fc1') as scope:
weights = _variable_with_weight_decay(
'weights',
shape=[input_size, layer1_size],
stddev=0.04,
wd=0.004)
biases = _variable_on_cpu('biases', [layer1_size],
tf.constant_initializer(0.1))
pruned_weights = pruning.apply_mask(weights, scope)
_mask_summary(pruned_weights)
fc1 = tf.nn.relu(tf.matmul(input_layer, pruned_weights) + biases,
name=scope.name)
_activation_summary(fc1)
# logits layer
with tf.variable_scope('logits') as scope:
weights = _variable_with_weight_decay('weights', [layer1_size, 10],
stddev=1 / 192.0,
wd=0.0)
biases = _variable_on_cpu('biases', [10],
tf.constant_initializer(0.0))
pruned_weights = pruning.apply_mask(weights, scope)
_mask_summary(pruned_weights)
logits = tf.add(tf.matmul(fc1, pruned_weights),
biases,
name=scope.name)
_activation_summary(logits)
return estimator_fn_complete(pruning_hparam_args, features, labels,
mode, logits)
def conv_layer(last_layer, shape, scope):
kernel = _variable_with_weight_decay('weights',
shape=shape,
stddev=5e-2,
wd=0.0)
pruned_kernel = pruning.apply_mask(kernel, scope)
_mask_summary(pruned_kernel)
conv = tf.nn.conv2d(last_layer,
pruned_kernel, [1, 1, 1, 1],
padding='SAME')
biases = _variable_on_cpu('biases', [shape[3]],
tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases)
conv = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv)
return conv
def inference(images):
"""Build the CIFAR-10 model.
Args:
images: Images returned from distorted_inputs() or inputs().
Returns:
Logits.
"""
# We instantiate all variables using tf.get_variable() instead of
# tf.Variable() in order to share variables across multiple GPU training runs.
# If we only ran this model on a single GPU, we could simplify this function
# by replacing all instances of tf.get_variable() with tf.Variable().
#
# While instantiating conv and local layers, we add mask and threshold
# variables to the layer by calling the pruning.apply_mask() function.
# Note that the masks are applied only to the weight tensors
# conv1
l1_kernels = 64
l2_kernels = 64
with tf.variable_scope('conv1') as scope:
kernel = _variable_with_weight_decay(
'weights', shape=[5, 5, 3, l1_kernels], stddev=5e-2, wd=0.0)
conv = tf.nn.conv2d(
images, pruning.apply_mask(kernel, scope), [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [l1_kernels], tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv1)
# pool1
pool1 = tf.nn.max_pool(
conv1,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# norm1
norm1 = tf.nn.lrn(
pool1, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm1')
# conv2
with tf.variable_scope('conv2') as scope:
kernel = _variable_with_weight_decay(
'weights', shape=[5, 5, l1_kernels, l2_kernels], stddev=5e-2, wd=0.0)
conv = tf.nn.conv2d(
norm1, pruning.apply_mask(kernel, scope), [1, 1, 1, 1], padding='SAME')
biases = _variable_on_cpu('biases', [l2_kernels], tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv2)
#pool2
pool2 = tf.nn.max_pool(
conv2,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
# norm2
norm2 = tf.nn.lrn(
conv2, 4, bias=1.0, alpha=0.001 / 9.0, beta=0.75, name='norm2')
#
# local3
with tf.variable_scope('local3') as scope:
# Move everything into depth so we can perform a single matrix multiply.
reshape = tf.reshape(pool2, [BATCH_SIZE, -1])
dim = reshape.get_shape()[1].value
weights = _variable_with_weight_decay(
'weights', shape=[dim, 384], stddev=0.04, wd=0.004)
biases = _variable_on_cpu('biases', [384], tf.constant_initializer(0.1))
local3 = tf.nn.relu(
tf.matmul(reshape, pruning.apply_mask(weights, scope)) + biases,
name=scope.name)
_activation_summary(local3)
# local4
with tf.variable_scope('local4') as scope:
weights = _variable_with_weight_decay(
'weights', shape=[384, 192], stddev=0.04, wd=0.004)
biases = _variable_on_cpu('biases', [192], tf.constant_initializer(0.1))
local4 = tf.nn.relu(
tf.matmul(local3, pruning.apply_mask(weights, scope)) + biases,
name=scope.name)
_activation_summary(local4)
# linear layer(WX + b),
# We don't apply softmax here because
# tf.nn.sparse_softmax_cross_entropy_with_logits accepts the unscaled logits
# and performs the softmax internally for efficiency.
with tf.variable_scope('softmax_linear') as scope:
weights = _variable_with_weight_decay(
'weights', [192, NUM_CLASSES], stddev=1 / 192.0, wd=0.0)
biases = _variable_on_cpu('biases', [NUM_CLASSES],
tf.constant_initializer(0.0))
softmax_linear = tf.add(
tf.matmul(local4, pruning.apply_mask(weights, scope)),
biases,
name=scope.name)
_activation_summary(softmax_linear)
return softmax_linear
def lenet5_model_fn(features, labels, mode):
l1_filters = 20
l2_filters = 50
# Input Layer
input_layer = tf.reshape(features["x"], [-1, 28, 28, 1])
with tf.variable_scope('conv1') as scope:
kernel = _variable_with_weight_decay('weights',
shape=[5, 5, 1, l1_filters],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(input_layer,
pruning.apply_mask(kernel, scope),
[1, 1, 1, 1],
padding='SAME')
biases = _variable_on_cpu('biases', [l1_filters],
tf.constant_initializer(0.0))
pre_activation = tf.nn.bias_add(conv, biases)
conv1 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv1)
# pool1
pool1 = tf.nn.max_pool(conv1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool1')
# norm1
norm1 = tf.nn.lrn(pool1,
4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='norm1')
# conv2
with tf.variable_scope('conv2') as scope:
kernel = _variable_with_weight_decay(
'weights',
shape=[5, 5, l1_filters, l2_filters],
stddev=5e-2,
wd=0.0)
conv = tf.nn.conv2d(norm1,
pruning.apply_mask(kernel, scope),
[1, 1, 1, 1],
padding='SAME')
biases = _variable_on_cpu('biases', [l2_filters],
tf.constant_initializer(0.1))
pre_activation = tf.nn.bias_add(conv, biases)
conv2 = tf.nn.relu(pre_activation, name=scope.name)
_activation_summary(conv2)
# norm2
norm2 = tf.nn.lrn(conv2,
4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75,
name='norm2')
# pool2
pool2 = tf.nn.max_pool(norm2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME',
name='pool2')
pool2_flat_size = np.prod(pool2.shape.as_list()[1:])
# fully connected layer 1
with tf.variable_scope('fc1') as scope:
# Move everything into depth so we can perform a single matrix multiply.
reshape = tf.reshape(pool2, [-1, pool2_flat_size])
weights = _variable_with_weight_decay('weights',
shape=[pool2_flat_size, 384],
stddev=0.04,
wd=0.004)
biases = _variable_on_cpu('biases', [384],
tf.constant_initializer(0.1))
fc1 = tf.nn.relu(
tf.matmul(reshape, pruning.apply_mask(weights, scope)) +
biases,
name=scope.name)
_activation_summary(fc1)
# fully connected layer 2
with tf.variable_scope('logits') as scope:
weights = _variable_with_weight_decay('weights', [384, 10],
stddev=1 / 192.0,
wd=0.0)
biases = _variable_on_cpu('biases', [10],
tf.constant_initializer(0.0))
logits = tf.add(tf.matmul(fc1, pruning.apply_mask(weights, scope)),
biases,
name=scope.name)
_activation_summary(logits)
return estimator_fn_complete(pruning_hparam_args, features, labels,
mode, logits)