def _build_graph(self, inputs):
image, label = inputs
image = image / 128.0 - 1
with argscope(Conv2D, nl=BNReLU, use_bias=False):
logits = (LinearWrap(image)
.Conv2D('conv1', 24, 5, padding='VALID')
.MaxPooling('pool1', 2, padding='SAME')
.Conv2D('conv2', 32, 3, padding='VALID')
.Conv2D('conv3', 32, 3, padding='VALID')
.MaxPooling('pool2', 2, padding='SAME')
.Conv2D('conv4', 64, 3, padding='VALID')
.Dropout('drop', 0.5)
.FullyConnected('fc0', 512,
b_init=tf.constant_initializer(0.1), nl=tf.nn.relu)
.FullyConnected('linear', out_dim=10, nl=tf.identity)())
prob = tf.nn.softmax(logits, name='output')
# compute the number of failed samples, for ClassificationError to use at test time
wrong = prediction_incorrect(logits, label)
accuracy = symbf.accuracy(logits, label, name='accuracy')
# monitor training error
add_moving_summary(tf.reduce_mean(wrong, name='train_error'), accuracy)
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss')
wd_cost = regularize_cost('fc.*/W', l2_regularizer(0.00001))
add_moving_summary(cost, wd_cost)
add_param_summary(('.*/W', ['histogram', 'rms'])) # monitor W
self.cost = tf.add_n([cost, wd_cost], name='cost')
def _build_graph(self, inputs):
points, label = inputs
num_point = points.get_shape()[1]
logger.info('input transform')
trans_mat = self.input_transform(points)
points = tf.matmul(points, trans_mat)
points = tf.expand_dims(points, -1)
logger.info('mlp(64, 64)')
with argscope(Conv2D, nl=BNReLU, padding='VALID'):
points = (LinearWrap(points).Conv2D('conv0',
64,
kernel_shape=[1, 3]).Conv2D(
'conv1',
64,
kernel_shape=[1, 1])())
if N_TRANSFORM:
logger.info('feature transform')
trans_mat2 = self.feature_transform(points)
points = tf.reshape(points, [-1, NUM_POINTS, 64])
points = tf.matmul(points, trans_mat2)
points = tf.expand_dims(points, [2])
logger.info('mlp(64, 128, 1024)')
with argscope(Conv2D, nl=BNReLU, kernel_shape=[1, 1], padding='VALID'):
points = (LinearWrap(points).Conv2D('conv2', 64).Conv2D(
'conv3', 128).Conv2D('conv4', 1024)())
logger.info('global feature')
points = MaxPooling('tpool0', points, [num_point, 1])
logger.info('output scores')
with argscope([Conv2D, FullyConnected], nl=BNReLU):
logits = (LinearWrap(points).FullyConnected('fc0', 512).Dropout(
'drop0', 0.5).FullyConnected('fc1', 256).Dropout(
'drop1', 0.5).FullyConnected('fc2', 40, nl=tf.identity)())
# vanilla classification loss
cls_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=label)
cls_loss = tf.reduce_mean(cls_loss, name="cls_costs")
accuracy = symbf.accuracy(logits, label, name='accuracy')
if N_TRANSFORM:
# orthogonality
mat_diff = tf.matmul(trans_mat2,
tf.transpose(trans_mat2, perm=[0, 2, 1]))
mat_diff = tf.subtract(mat_diff,
tf.constant(np.eye(64), dtype=tf.float32),
name="mat_diff")
mat_diff_loss = tf.nn.l2_loss(mat_diff)
self.cost = tf.add(cls_loss, mat_diff_loss, name="total_costs")
summary.add_moving_summary(mat_diff_loss)
else:
self.cost = tf.identity(cls_loss, name="total_costs")
summary.add_moving_summary(cls_loss, self.cost, accuracy)
def _build_graph(self, inputs):
"""This function should build the model which takes the input variables
and define self.cost at the end"""
# inputs contains a list of input variables defined above
image, label = inputs
# In tensorflow, inputs to convolution function are assumed to be
# NHWC. Add a single channel here.
image = tf.expand_dims(image, 3)
image = image * 2 - 1 # center the pixels values at zero
# The context manager `argscope` sets the default option for all the layers under
# this context. Here we use 32 channel convolution with shape 3x3
with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32):
logits = (LinearWrap(image)
.Conv2D('conv0')
.MaxPooling('pool0', 2)
.Conv2D('conv1')
.Conv2D('conv2')
.MaxPooling('pool1', 2)
.Conv2D('conv3')
.FullyConnected('fc0', 512, nl=tf.nn.relu)
.Dropout('dropout', 0.5)
.FullyConnected('fc1', out_dim=10, nl=tf.identity)())
prob = tf.nn.softmax(logits, name='prob') # a Bx10 with probabilities
# a vector of length B with loss of each sample
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss') # the average cross-entropy loss
# compute the "incorrect vector", for the callback ClassificationError to use at validation time
wrong = symbf.prediction_incorrect(logits, label, name='incorrect')
accuracy = symbf.accuracy(logits, label, name='accuracy')
# This will monitor training error (in a moving_average fashion):
# 1. write the value to tensosrboard
# 2. write the value to stat.json
# 3. print the value after each epoch
train_error = tf.reduce_mean(wrong, name='train_error')
summary.add_moving_summary(train_error, accuracy)
# Use a regex to find parameters to apply weight decay.
# Here we apply a weight decay on all W (weight matrix) of all fc layers
wd_cost = tf.multiply(1e-5,
regularize_cost('fc.*/W', tf.nn.l2_loss),
name='regularize_loss')
self.cost = tf.add_n([wd_cost, cost], name='total_cost')
summary.add_moving_summary(cost, wd_cost, self.cost)
# monitor histogram of all weight (of conv and fc layers) in tensorboard
summary.add_param_summary(('.*/W', ['histogram', 'rms']))
def _build_graph(self, inputs):
image, label = inputs
is_training = get_current_tower_context().is_training
keep_prob = tf.constant(0.5 if is_training else 1.0)
if is_training:
tf.summary.image("train_image", image, 10)
if tf.test.is_gpu_available():
image = tf.transpose(image, [0, 3, 1, 2])
data_format = 'NCHW'
else:
data_format = 'NHWC'
image = image / 4.0 # just to make range smaller
with argscope(Conv2D, nl=BNReLU, use_bias=False, kernel_shape=3), \
argscope([Conv2D, MaxPooling, BatchNorm], data_format=data_format):
logits = LinearWrap(image) \
.Conv2D('conv1.1', out_channel=64) \
.Conv2D('conv1.2', out_channel=64) \
.MaxPooling('pool1', 3, stride=2, padding='SAME') \
.Conv2D('conv2.1', out_channel=128) \
.Conv2D('conv2.2', out_channel=128) \
.MaxPooling('pool2', 3, stride=2, padding='SAME') \
.Conv2D('conv3.1', out_channel=128, padding='VALID') \
.Conv2D('conv3.2', out_channel=128, padding='VALID') \
.FullyConnected('fc0', 1024 + 512, nl=tf.nn.relu) \
.tf.nn.dropout(keep_prob) \
.FullyConnected('fc1', 512, nl=tf.nn.relu) \
.FullyConnected('linear', out_dim=self.cifar_classnum, nl=tf.identity)()
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss')
wrong = symbf.prediction_incorrect(logits, label)
accuracy = symbf.accuracy(logits, label, name='accuracy')
# monitor training error
add_moving_summary(tf.reduce_mean(wrong, name='train_error'), accuracy)
# weight decay on all W of fc layers
wd_cost = regularize_cost('fc.*/W',
l2_regularizer(4e-4),
name='regularize_loss')
add_moving_summary(cost, wd_cost)
add_param_summary(('.*/W', ['histogram'])) # monitor W
self.cost = tf.add_n([cost, wd_cost], name='cost')
def _build_graph(self, inputs):
image, label = inputs
image = tf.expand_dims(image * 2 - 1, 3)
with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32):
c0 = Conv2D('conv0', image)
p0 = MaxPooling('pool0', c0, 2)
c1 = Conv2D('conv1', p0)
c2 = Conv2D('conv2', c1)
p1 = MaxPooling('pool1', c2, 2)
c3 = Conv2D('conv3', p1)
fc1 = FullyConnected('fc0', c3, 512, nl=tf.nn.relu)
fc1 = Dropout('dropout', fc1, 0.5)
logits = FullyConnected('fc1', fc1, out_dim=10, nl=tf.identity)
with tf.name_scope('visualizations'):
visualize_conv_weights(c0.variables.W, 'conv0')
visualize_conv_activations(c0, 'conv0')
visualize_conv_weights(c1.variables.W, 'conv1')
visualize_conv_activations(c1, 'conv1')
visualize_conv_weights(c2.variables.W, 'conv2')
visualize_conv_activations(c2, 'conv2')
visualize_conv_weights(c3.variables.W, 'conv3')
visualize_conv_activations(c3, 'conv3')
tf.summary.image('input', (image + 1.0) * 128., 3)
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss')
wrong = symbf.prediction_incorrect(logits, label, name='incorrect')
accuracy = symbf.accuracy(logits, label)
wd_cost = tf.multiply(1e-5,
regularize_cost('fc.*/W', tf.nn.l2_loss),
name='regularize_loss')
self.cost = tf.add_n([wd_cost, cost], name='total_cost')
summary.add_moving_summary(cost, wd_cost, self.cost, accuracy)
summary.add_param_summary(('.*/W', ['histogram', 'rms']))
def _build_graph(self, inputs):
image, label = inputs
fw, fa, fg = get_dorefa(BITW, BITA, BITG)
old_get_variable = tf.get_variable
def monitor(x, name):
if MONITOR == 1:
return tf.Print(x, [x],
message='\n\n' + name + ': ',
summarize=1000,
name=name)
else:
return x
def new_get_variable(v):
name = v.op.name
if not name.endswith('W') or 'conv1_1' in name or 'fc8' in name:
return v
else:
logger.info("Quantizing weight {}".format(v.op.name))
if MONITOR == 1:
return tf.Print(fw(v), [fw(v)],
message='\n\n' + v.name +
', Quantized weights are:',
summarize=100)
else:
return fw(v)
def bn_activate(name, x):
X = BatchNorm(name, x)
x = monitor(x, name + '_noact_out')
return activate(x)
def activate(x):
if BITA == 32:
return tf.nn.relu(x)
else:
return fa(tf.nn.relu(x))
# VGG 16
with remap_variables(new_get_variable), \
argscope(Conv2D, kernel_shape=3, use_bias=False, nl = tf.identity):
logits = (
LinearWrap(image).apply(monitor, 'image_out').Conv2D(
'conv1_1',
64).apply(fg).BatchNorm('bn1_1').apply(activate).apply(
monitor, 'conv1_1_out').Conv2D('conv1_2', 64).apply(
fg).BatchNorm('bn1_2').apply(activate).apply(
monitor,
'conv1_2_out').MaxPooling('pool1', 2).apply(
monitor, 'pool1_out')
# 112
.Conv2D(
'conv2_1',
128).apply(fg).BatchNorm('bn2_1').apply(activate).apply(
monitor, 'conv2_1_out').Conv2D('conv2_2', 128).apply(
fg).BatchNorm('bn2_2').apply(activate).apply(
monitor, 'conv2_2_out').MaxPooling(
'pool2', 2).apply(monitor, 'pool2_out')
# 56
.Conv2D(
'conv3_1',
256).apply(fg).BatchNorm('bn3_1').apply(activate).apply(
monitor, 'conv3_1_out').Conv2D(
'conv3_2', 256).apply(fg).BatchNorm('bn3_2').
apply(activate).apply(monitor, 'conv3_2_out').Conv2D(
'conv3_3',
256).apply(fg).BatchNorm('bn3_3').apply(activate).apply(
monitor, 'conv3_3_out').MaxPooling('pool3', 2).apply(
monitor, 'pool3_out')
# 28
.Conv2D(
'conv4_1',
512).apply(fg).BatchNorm('bn4_1').apply(activate).apply(
monitor, 'conv4_1_out').Conv2D(
'conv4_2', 512).apply(fg).BatchNorm('bn4_2').
apply(activate).apply(monitor, 'conv4_2_out').Conv2D(
'conv4_3',
512).apply(fg).BatchNorm('bn4_3').apply(activate).apply(
monitor, 'conv4_3_out').MaxPooling('pool4', 2).apply(
monitor, 'pool4_out')
# 14
.Conv2D(
'conv5_1',
512).apply(fg).BatchNorm('bn5_1').apply(activate).apply(
monitor, 'conv5_1_out').Conv2D(
'conv5_2', 512).apply(fg).BatchNorm('bn5_2').
apply(activate).apply(monitor, 'conv5_2_out').Conv2D(
'conv5_3',
512).apply(fg).BatchNorm('bn5_3').apply(activate).apply(
monitor, 'conv5_3_out').MaxPooling('pool5', 2).apply(
monitor, 'pool5_out').FullyConnected(
'fc6', use_bias=False,
out_dim=512).apply(activate).apply(
monitor, 'fc6_out').FullyConnected(
'fc7', use_bias=False,
out_dim=512).apply(activate).apply(
monitor, 'fc7_out').FullyConnected(
'fc8',
use_bias=False,
out_dim=self.cifar_classnum,
nl=tf.identity).apply(
monitor, 'fc8_out')())
prob = tf.nn.softmax(logits, name='prob')
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss')
wrong = symbf.prediction_incorrect(logits, label, name='incorrect')
accuracy = symbf.accuracy(logits, label, name='accuracy')
train_error = tf.reduce_mean(wrong, name='train_error')
summary.add_moving_summary(train_error, accuracy)
wd_cost = tf.multiply(1e-5,
regularize_cost('fc.*/W', tf.nn.l2_loss),
name='regularize_loss')
self.cost = tf.add_n([wd_cost, cost], name='total_cost')
summary.add_moving_summary(cost, wd_cost, self.cost)
def _build_graph(self, inputs):
image, label = inputs
"""Add a single channel here"""
image = tf.expand_dims(image, 3)
image = image * 256
image = tf.round(image)
fw, fa, fg = get_dorefa(BITW, BITA, BITG)
old_get_variable = tf.get_variable
def monitor(x, name):
if MONITOR == 1:
return tf.Print(x, [x],
message='\n\n' + name + ': ',
summarize=1000,
name=name)
else:
return x
def new_get_variable(v):
name = v.op.name
if not name.endswith('W') or 'conv0' in name or 'fc1' in name:
return v
else:
logger.info("Quantizing weight {}".format(v.op.name))
if MONITOR == 1:
return tf.Print(fw(v), [fw(v)],
message='\n\n' + v.name +
', Quantized weights are:',
summarize=100)
else:
return fw(v)
def activate(x):
if BITA == 32:
return tf.nn.relu(x)
else:
return fa(tf.nn.relu(x))
with remap_variables(new_get_variable), \
argscope(Conv2D, kernel_shape=3, use_bias=False, nl=tf.identity, out_channel=32):
logits = (LinearWrap(image).apply(monitor, 'image_out').Conv2D(
'conv0').apply(fg).BatchNorm('bn0').apply(activate).apply(
monitor, 'conv0_out').MaxPooling('pool0', 2).apply(
monitor, 'pool0_out').Conv2D('conv1').apply(
fg).BatchNorm('bn1').apply(activate).apply(
monitor, 'conv1_out').Conv2D('conv2').apply(
fg).BatchNorm('bn2').apply(activate).apply(
monitor, 'conv2_out').MaxPooling(
'pool1', 2).apply(
monitor,
'pool1_out').Conv2D('conv3').
apply(fg).BatchNorm('bn3').apply(activate).apply(
monitor, 'conv3_out').FullyConnected(
'fc0',
use_bias=False,
out_dim=20,
nl=tf.identity).apply(activate).apply(
monitor, 'fc0_out').FullyConnected(
'fc1',
use_bias=False,
out_dim=10,
nl=tf.identity).apply(
monitor, 'fc1_out')())
prob = tf.nn.softmax(logits, name='prob')
cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=label)
cost = tf.reduce_mean(cost, name='cross_entropy_loss')
wrong = symbf.prediction_incorrect(logits, label, name='incorrect')
accuracy = symbf.accuracy(logits, label, name='accuracy')
train_error = tf.reduce_mean(wrong, name='train_error')
summary.add_moving_summary(train_error, accuracy)
wd_cost = tf.multiply(1e-5,
regularize_cost('fc.*/W', tf.nn.l2_loss),
name='regularize_loss')
self.cost = tf.add_n([wd_cost, cost], name='total_cost')
summary.add_moving_summary(cost, wd_cost, self.cost)
