def _build_graph(self, inputs): image, label = inputs image = image / 128.0 - 1 with argscope(Conv2D, nl=BNReLU, use_bias=False): logits = (LinearWrap(image) .Conv2D('conv1', 24, 5, padding='VALID') .MaxPooling('pool1', 2, padding='SAME') .Conv2D('conv2', 32, 3, padding='VALID') .Conv2D('conv3', 32, 3, padding='VALID') .MaxPooling('pool2', 2, padding='SAME') .Conv2D('conv4', 64, 3, padding='VALID') .Dropout('drop', 0.5) .FullyConnected('fc0', 512, b_init=tf.constant_initializer(0.1), nl=tf.nn.relu) .FullyConnected('linear', out_dim=10, nl=tf.identity)()) prob = tf.nn.softmax(logits, name='output') # compute the number of failed samples, for ClassificationError to use at test time wrong = prediction_incorrect(logits, label) accuracy = symbf.accuracy(logits, label, name='accuracy') # monitor training error add_moving_summary(tf.reduce_mean(wrong, name='train_error'), accuracy) cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label) cost = tf.reduce_mean(cost, name='cross_entropy_loss') wd_cost = regularize_cost('fc.*/W', l2_regularizer(0.00001)) add_moving_summary(cost, wd_cost) add_param_summary(('.*/W', ['histogram', 'rms'])) # monitor W self.cost = tf.add_n([cost, wd_cost], name='cost')
def _build_graph(self, inputs): points, label = inputs num_point = points.get_shape()[1] logger.info('input transform') trans_mat = self.input_transform(points) points = tf.matmul(points, trans_mat) points = tf.expand_dims(points, -1) logger.info('mlp(64, 64)') with argscope(Conv2D, nl=BNReLU, padding='VALID'): points = (LinearWrap(points).Conv2D('conv0', 64, kernel_shape=[1, 3]).Conv2D( 'conv1', 64, kernel_shape=[1, 1])()) if N_TRANSFORM: logger.info('feature transform') trans_mat2 = self.feature_transform(points) points = tf.reshape(points, [-1, NUM_POINTS, 64]) points = tf.matmul(points, trans_mat2) points = tf.expand_dims(points, [2]) logger.info('mlp(64, 128, 1024)') with argscope(Conv2D, nl=BNReLU, kernel_shape=[1, 1], padding='VALID'): points = (LinearWrap(points).Conv2D('conv2', 64).Conv2D( 'conv3', 128).Conv2D('conv4', 1024)()) logger.info('global feature') points = MaxPooling('tpool0', points, [num_point, 1]) logger.info('output scores') with argscope([Conv2D, FullyConnected], nl=BNReLU): logits = (LinearWrap(points).FullyConnected('fc0', 512).Dropout( 'drop0', 0.5).FullyConnected('fc1', 256).Dropout( 'drop1', 0.5).FullyConnected('fc2', 40, nl=tf.identity)()) # vanilla classification loss cls_loss = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=logits, labels=label) cls_loss = tf.reduce_mean(cls_loss, name="cls_costs") accuracy = symbf.accuracy(logits, label, name='accuracy') if N_TRANSFORM: # orthogonality mat_diff = tf.matmul(trans_mat2, tf.transpose(trans_mat2, perm=[0, 2, 1])) mat_diff = tf.subtract(mat_diff, tf.constant(np.eye(64), dtype=tf.float32), name="mat_diff") mat_diff_loss = tf.nn.l2_loss(mat_diff) self.cost = tf.add(cls_loss, mat_diff_loss, name="total_costs") summary.add_moving_summary(mat_diff_loss) else: self.cost = tf.identity(cls_loss, name="total_costs") summary.add_moving_summary(cls_loss, self.cost, accuracy)
def _build_graph(self, inputs): """This function should build the model which takes the input variables and define self.cost at the end""" # inputs contains a list of input variables defined above image, label = inputs # In tensorflow, inputs to convolution function are assumed to be # NHWC. Add a single channel here. image = tf.expand_dims(image, 3) image = image * 2 - 1 # center the pixels values at zero # The context manager `argscope` sets the default option for all the layers under # this context. Here we use 32 channel convolution with shape 3x3 with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32): logits = (LinearWrap(image) .Conv2D('conv0') .MaxPooling('pool0', 2) .Conv2D('conv1') .Conv2D('conv2') .MaxPooling('pool1', 2) .Conv2D('conv3') .FullyConnected('fc0', 512, nl=tf.nn.relu) .Dropout('dropout', 0.5) .FullyConnected('fc1', out_dim=10, nl=tf.identity)()) prob = tf.nn.softmax(logits, name='prob') # a Bx10 with probabilities # a vector of length B with loss of each sample cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label) cost = tf.reduce_mean(cost, name='cross_entropy_loss') # the average cross-entropy loss # compute the "incorrect vector", for the callback ClassificationError to use at validation time wrong = symbf.prediction_incorrect(logits, label, name='incorrect') accuracy = symbf.accuracy(logits, label, name='accuracy') # This will monitor training error (in a moving_average fashion): # 1. write the value to tensosrboard # 2. write the value to stat.json # 3. print the value after each epoch train_error = tf.reduce_mean(wrong, name='train_error') summary.add_moving_summary(train_error, accuracy) # Use a regex to find parameters to apply weight decay. # Here we apply a weight decay on all W (weight matrix) of all fc layers wd_cost = tf.multiply(1e-5, regularize_cost('fc.*/W', tf.nn.l2_loss), name='regularize_loss') self.cost = tf.add_n([wd_cost, cost], name='total_cost') summary.add_moving_summary(cost, wd_cost, self.cost) # monitor histogram of all weight (of conv and fc layers) in tensorboard summary.add_param_summary(('.*/W', ['histogram', 'rms']))
def _build_graph(self, inputs): image, label = inputs is_training = get_current_tower_context().is_training keep_prob = tf.constant(0.5 if is_training else 1.0) if is_training: tf.summary.image("train_image", image, 10) if tf.test.is_gpu_available(): image = tf.transpose(image, [0, 3, 1, 2]) data_format = 'NCHW' else: data_format = 'NHWC' image = image / 4.0 # just to make range smaller with argscope(Conv2D, nl=BNReLU, use_bias=False, kernel_shape=3), \ argscope([Conv2D, MaxPooling, BatchNorm], data_format=data_format): logits = LinearWrap(image) \ .Conv2D('conv1.1', out_channel=64) \ .Conv2D('conv1.2', out_channel=64) \ .MaxPooling('pool1', 3, stride=2, padding='SAME') \ .Conv2D('conv2.1', out_channel=128) \ .Conv2D('conv2.2', out_channel=128) \ .MaxPooling('pool2', 3, stride=2, padding='SAME') \ .Conv2D('conv3.1', out_channel=128, padding='VALID') \ .Conv2D('conv3.2', out_channel=128, padding='VALID') \ .FullyConnected('fc0', 1024 + 512, nl=tf.nn.relu) \ .tf.nn.dropout(keep_prob) \ .FullyConnected('fc1', 512, nl=tf.nn.relu) \ .FullyConnected('linear', out_dim=self.cifar_classnum, nl=tf.identity)() cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label) cost = tf.reduce_mean(cost, name='cross_entropy_loss') wrong = symbf.prediction_incorrect(logits, label) accuracy = symbf.accuracy(logits, label, name='accuracy') # monitor training error add_moving_summary(tf.reduce_mean(wrong, name='train_error'), accuracy) # weight decay on all W of fc layers wd_cost = regularize_cost('fc.*/W', l2_regularizer(4e-4), name='regularize_loss') add_moving_summary(cost, wd_cost) add_param_summary(('.*/W', ['histogram'])) # monitor W self.cost = tf.add_n([cost, wd_cost], name='cost')
def _build_graph(self, inputs): image, label = inputs image = tf.expand_dims(image * 2 - 1, 3) with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu, out_channel=32): c0 = Conv2D('conv0', image) p0 = MaxPooling('pool0', c0, 2) c1 = Conv2D('conv1', p0) c2 = Conv2D('conv2', c1) p1 = MaxPooling('pool1', c2, 2) c3 = Conv2D('conv3', p1) fc1 = FullyConnected('fc0', c3, 512, nl=tf.nn.relu) fc1 = Dropout('dropout', fc1, 0.5) logits = FullyConnected('fc1', fc1, out_dim=10, nl=tf.identity) with tf.name_scope('visualizations'): visualize_conv_weights(c0.variables.W, 'conv0') visualize_conv_activations(c0, 'conv0') visualize_conv_weights(c1.variables.W, 'conv1') visualize_conv_activations(c1, 'conv1') visualize_conv_weights(c2.variables.W, 'conv2') visualize_conv_activations(c2, 'conv2') visualize_conv_weights(c3.variables.W, 'conv3') visualize_conv_activations(c3, 'conv3') tf.summary.image('input', (image + 1.0) * 128., 3) cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label) cost = tf.reduce_mean(cost, name='cross_entropy_loss') wrong = symbf.prediction_incorrect(logits, label, name='incorrect') accuracy = symbf.accuracy(logits, label) wd_cost = tf.multiply(1e-5, regularize_cost('fc.*/W', tf.nn.l2_loss), name='regularize_loss') self.cost = tf.add_n([wd_cost, cost], name='total_cost') summary.add_moving_summary(cost, wd_cost, self.cost, accuracy) summary.add_param_summary(('.*/W', ['histogram', 'rms']))
def _build_graph(self, inputs): image, label = inputs fw, fa, fg = get_dorefa(BITW, BITA, BITG) old_get_variable = tf.get_variable def monitor(x, name): if MONITOR == 1: return tf.Print(x, [x], message='\n\n' + name + ': ', summarize=1000, name=name) else: return x def new_get_variable(v): name = v.op.name if not name.endswith('W') or 'conv1_1' in name or 'fc8' in name: return v else: logger.info("Quantizing weight {}".format(v.op.name)) if MONITOR == 1: return tf.Print(fw(v), [fw(v)], message='\n\n' + v.name + ', Quantized weights are:', summarize=100) else: return fw(v) def bn_activate(name, x): X = BatchNorm(name, x) x = monitor(x, name + '_noact_out') return activate(x) def activate(x): if BITA == 32: return tf.nn.relu(x) else: return fa(tf.nn.relu(x)) # VGG 16 with remap_variables(new_get_variable), \ argscope(Conv2D, kernel_shape=3, use_bias=False, nl = tf.identity): logits = ( LinearWrap(image).apply(monitor, 'image_out').Conv2D( 'conv1_1', 64).apply(fg).BatchNorm('bn1_1').apply(activate).apply( monitor, 'conv1_1_out').Conv2D('conv1_2', 64).apply( fg).BatchNorm('bn1_2').apply(activate).apply( monitor, 'conv1_2_out').MaxPooling('pool1', 2).apply( monitor, 'pool1_out') # 112 .Conv2D( 'conv2_1', 128).apply(fg).BatchNorm('bn2_1').apply(activate).apply( monitor, 'conv2_1_out').Conv2D('conv2_2', 128).apply( fg).BatchNorm('bn2_2').apply(activate).apply( monitor, 'conv2_2_out').MaxPooling( 'pool2', 2).apply(monitor, 'pool2_out') # 56 .Conv2D( 'conv3_1', 256).apply(fg).BatchNorm('bn3_1').apply(activate).apply( monitor, 'conv3_1_out').Conv2D( 'conv3_2', 256).apply(fg).BatchNorm('bn3_2'). apply(activate).apply(monitor, 'conv3_2_out').Conv2D( 'conv3_3', 256).apply(fg).BatchNorm('bn3_3').apply(activate).apply( monitor, 'conv3_3_out').MaxPooling('pool3', 2).apply( monitor, 'pool3_out') # 28 .Conv2D( 'conv4_1', 512).apply(fg).BatchNorm('bn4_1').apply(activate).apply( monitor, 'conv4_1_out').Conv2D( 'conv4_2', 512).apply(fg).BatchNorm('bn4_2'). apply(activate).apply(monitor, 'conv4_2_out').Conv2D( 'conv4_3', 512).apply(fg).BatchNorm('bn4_3').apply(activate).apply( monitor, 'conv4_3_out').MaxPooling('pool4', 2).apply( monitor, 'pool4_out') # 14 .Conv2D( 'conv5_1', 512).apply(fg).BatchNorm('bn5_1').apply(activate).apply( monitor, 'conv5_1_out').Conv2D( 'conv5_2', 512).apply(fg).BatchNorm('bn5_2'). apply(activate).apply(monitor, 'conv5_2_out').Conv2D( 'conv5_3', 512).apply(fg).BatchNorm('bn5_3').apply(activate).apply( monitor, 'conv5_3_out').MaxPooling('pool5', 2).apply( monitor, 'pool5_out').FullyConnected( 'fc6', use_bias=False, out_dim=512).apply(activate).apply( monitor, 'fc6_out').FullyConnected( 'fc7', use_bias=False, out_dim=512).apply(activate).apply( monitor, 'fc7_out').FullyConnected( 'fc8', use_bias=False, out_dim=self.cifar_classnum, nl=tf.identity).apply( monitor, 'fc8_out')()) prob = tf.nn.softmax(logits, name='prob') cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label) cost = tf.reduce_mean(cost, name='cross_entropy_loss') wrong = symbf.prediction_incorrect(logits, label, name='incorrect') accuracy = symbf.accuracy(logits, label, name='accuracy') train_error = tf.reduce_mean(wrong, name='train_error') summary.add_moving_summary(train_error, accuracy) wd_cost = tf.multiply(1e-5, regularize_cost('fc.*/W', tf.nn.l2_loss), name='regularize_loss') self.cost = tf.add_n([wd_cost, cost], name='total_cost') summary.add_moving_summary(cost, wd_cost, self.cost)
def _build_graph(self, inputs): image, label = inputs """Add a single channel here""" image = tf.expand_dims(image, 3) image = image * 256 image = tf.round(image) fw, fa, fg = get_dorefa(BITW, BITA, BITG) old_get_variable = tf.get_variable def monitor(x, name): if MONITOR == 1: return tf.Print(x, [x], message='\n\n' + name + ': ', summarize=1000, name=name) else: return x def new_get_variable(v): name = v.op.name if not name.endswith('W') or 'conv0' in name or 'fc1' in name: return v else: logger.info("Quantizing weight {}".format(v.op.name)) if MONITOR == 1: return tf.Print(fw(v), [fw(v)], message='\n\n' + v.name + ', Quantized weights are:', summarize=100) else: return fw(v) def activate(x): if BITA == 32: return tf.nn.relu(x) else: return fa(tf.nn.relu(x)) with remap_variables(new_get_variable), \ argscope(Conv2D, kernel_shape=3, use_bias=False, nl=tf.identity, out_channel=32): logits = (LinearWrap(image).apply(monitor, 'image_out').Conv2D( 'conv0').apply(fg).BatchNorm('bn0').apply(activate).apply( monitor, 'conv0_out').MaxPooling('pool0', 2).apply( monitor, 'pool0_out').Conv2D('conv1').apply( fg).BatchNorm('bn1').apply(activate).apply( monitor, 'conv1_out').Conv2D('conv2').apply( fg).BatchNorm('bn2').apply(activate).apply( monitor, 'conv2_out').MaxPooling( 'pool1', 2).apply( monitor, 'pool1_out').Conv2D('conv3'). apply(fg).BatchNorm('bn3').apply(activate).apply( monitor, 'conv3_out').FullyConnected( 'fc0', use_bias=False, out_dim=20, nl=tf.identity).apply(activate).apply( monitor, 'fc0_out').FullyConnected( 'fc1', use_bias=False, out_dim=10, nl=tf.identity).apply( monitor, 'fc1_out')()) prob = tf.nn.softmax(logits, name='prob') cost = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=label) cost = tf.reduce_mean(cost, name='cross_entropy_loss') wrong = symbf.prediction_incorrect(logits, label, name='incorrect') accuracy = symbf.accuracy(logits, label, name='accuracy') train_error = tf.reduce_mean(wrong, name='train_error') summary.add_moving_summary(train_error, accuracy) wd_cost = tf.multiply(1e-5, regularize_cost('fc.*/W', tf.nn.l2_loss), name='regularize_loss') self.cost = tf.add_n([wd_cost, cost], name='total_cost') summary.add_moving_summary(cost, wd_cost, self.cost)