def get_logits(self, image): with argscope([Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm], data_format=self.data_format): with argscope([BatchNorm], use_local_stat=False): return resnet_backbone( image, self.num_blocks, preresnet_group if self.mode == 'preact' else resnet_group, resnet_group_dilation, self.block_func, resnet_bottleneck_dilation)#Fintune or Joint Train
def get_logits(self, image): with argscope([Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format=self.data_format), \ argscope(Conv2D, use_bias=False): # See Table 1 & 2 in https://arxiv.org/abs/1707.01083 group = args.group channels = { 3: [240, 480, 960], 4: [272, 544, 1088], 8: [384, 768, 1536] } mul = group * 4 # #chan has to be a multiple of this number channels = [int(math.ceil(x * args.ratio / mul) * mul) for x in channels[group]] # The first channel must be a multiple of group first_chan = int(math.ceil(24 * args.ratio / group) * group) logger.info("#Channels: " + str([first_chan] + channels)) l = Conv2D('conv1', image, first_chan, 3, strides=2, activation=BNReLU) l = MaxPooling('pool1', l, 3, 2, padding='SAME') l = shufflenet_stage('group1', l, channels[0], 4, group) l = shufflenet_stage('group2', l, channels[1], 8, group) l = shufflenet_stage('group3', l, channels[2], 4, group) l = GlobalAvgPooling('gap', l) logits = FullyConnected('linear', l, 1000) return logits
def get_logits(self, image): # image is a float32 (N, 3, size, size) print(image) print(image.shape) group_func = resnet_group block_func = resnet_bottleneck num_blocks = [3, 4, 6, 3] with argscope( [Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm], data_format='NCHW'), \ argscope(Conv2D, nl=tf.identity, use_bias=False, W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')): logits = (LinearWrap(image).Conv2D( 'conv0', 64, 7, stride=2, nl=BNReLU).MaxPooling( 'pool0', shape=3, stride=2, padding='SAME').GlobalAvgPooling('gap').FullyConnected( 'linear', 1000, nl=tf.identity)()) # logits is just a tensor! (batch_size x 1000) # logits = tf.stack([logits, logits, logits], axis=1) return logits
def get_logits(self, image): gauss_init = tf.random_normal_initializer(stddev=0.01) with argscope(Conv2D, kernel_initializer=tf.variance_scaling_initializer(scale=2.)), \ argscope([Conv2D, FullyConnected], activation=tf.nn.relu), \ argscope([Conv2D, MaxPooling], data_format='channels_last'): # necessary padding to get 55x55 after conv1 image = tf.pad(image, [[0, 0], [2, 2], [2, 2], [0, 0]]) l = Conv2D('conv1', image, filters=96, kernel_size=11, strides=4, padding='VALID') # size: 55 visualize_conv1_weights(l.variables.W) l = tf.nn.lrn(l, 2, bias=1.0, alpha=2e-5, beta=0.75, name='norm1') l = MaxPooling('pool1', l, 3, strides=2, padding='VALID') # 27 l = Conv2D('conv2', l, filters=256, kernel_size=5, split=2) l = tf.nn.lrn(l, 2, bias=1.0, alpha=2e-5, beta=0.75, name='norm2') l = MaxPooling('pool2', l, 3, strides=2, padding='VALID') # 13 l = Conv2D('conv3', l, filters=384, kernel_size=3) l = Conv2D('conv4', l, filters=384, kernel_size=3, split=2) l = Conv2D('conv5', l, filters=256, kernel_size=3, split=2) l = MaxPooling('pool3', l, 3, strides=2, padding='VALID') l = FullyConnected('fc6', l, 4096, kernel_initializer=gauss_init, bias_initializer=tf.ones_initializer()) l = Dropout(l, rate=0.5) l = FullyConnected('fc7', l, 4096, kernel_initializer=gauss_init) l = Dropout(l, rate=0.5) logits = FullyConnected('fc8', l, 1000, kernel_initializer=gauss_init) return logits
def ssdnet_backbone(image, **kwargs): # with ssdnet_argscope(): l = image #tf.transpose(image, perm=[0, 2, 3, 1]) with argscope([BatchNorm], training=False): # conv1 l = Conv2D('conv1', l, 24, 4, strides=2, activation=None, padding='SAME') with tf.variable_scope('conv1'): l = BNReLU(tf.concat([l, -l], axis=-1)) l = MaxPooling('pool1', l, 2) l = tf.stop_gradient(l) with argscope([BatchNorm], training=None): # conv2 l = LinearBottleneck('conv2', l, 48, 24, 3, t=1, use_ab=True) l = l + LinearBottleneck('conv3', l, 24, 24, 5, t=2, use_ab=True) ch_all = [48, 72, 96] iters = [2, 4, 4] mults = [3, 4, 6] hlist = [] for ii, (ch, it, mu) in enumerate(zip(ch_all, iters, mults)): use_ab = (ii < 2) for jj in range(it): name = 'inc{}/{}'.format(ii, jj) stride = 2 if jj == 0 else 1 k = 3 if jj < (it // 2) else 5 swap_block = True if jj % 2 == 1 else False l = inception(name, l, ch, k, stride, t=mu, swap_block=swap_block, use_ab=use_ab) hlist.append(l) return hlist
def backbone_scope(freeze): """ Args: freeze (bool): whether to freeze all the variables under the scope """ def nonlin(x): x = get_norm()(x) return tf.nn.relu(x) with argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'), \ argscope(Conv2D, use_bias=False, activation=nonlin, kernel_initializer=tf.variance_scaling_initializer( scale=2.0, mode='fan_out')), \ ExitStack() as stack: if cfg.BACKBONE.NORM in ['FreezeBN', 'SyncBN']: if freeze or cfg.BACKBONE.NORM == 'FreezeBN': stack.enter_context(argscope(BatchNorm, training=False)) else: stack.enter_context(argscope( BatchNorm, sync_statistics='nccl' if cfg.TRAINER == 'replicated' else 'horovod')) if freeze: stack.enter_context(freeze_variables(stop_gradient=False, skip_collection=True)) else: # the layers are not completely freezed, but we may want to only freeze the affine if cfg.BACKBONE.FREEZE_AFFINE: stack.enter_context(custom_getter_scope(freeze_affine_getter)) yield
def vgg16_backbone(image): with argscope(Conv2D, kernel_initializer=tf.variance_scaling_initializer(scale=2.)), \ argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'): x = convnormrelu(image, 'conv1_1', 64) x = convnormrelu(x, 'conv1_2', 64) x = MaxPooling('pool1', x, 2, strides=2, padding='VALID') x = convnormrelu(x, 'conv2_1', 128) x = convnormrelu(x, 'conv2_2', 128) x = MaxPooling('pool2', x, 2, strides=2, padding='VALID') x = convnormrelu(x, 'conv3_1', 256) x = convnormrelu(x, 'conv3_2', 256) x = convnormrelu(x, 'conv3_3', 256) x = MaxPooling('pool3', x, 2, strides=2, padding='VALID') x = convnormrelu(x, 'conv4_1', 512) x = convnormrelu(x, 'conv4_2', 512) x = convnormrelu(x, 'conv4_3', 512) x = MaxPooling('pool4', x, 2, strides=2, padding='VALID') x = convnormrelu(x, 'conv5_1', 512) x = convnormrelu(x, 'conv5_2', 512) x = convnormrelu(x, 'conv5_3', 512) x = MaxPooling('pool5', x, 2, strides=2, padding='VALID') return x
def get_logits(self, inputs): sc = ghostnet_arg_scope(data_format=self.data_format, weight_decay=self.weight_decay, use_batch_norm=True, batch_norm_decay=0.9997, batch_norm_epsilon=0.001, regularize_depthwise=False) with slim.arg_scope(sc): with argscope(Conv2D, kernel_initializer=kernel_initializer): with argscope([Conv2D, BatchNorm], data_format=self.data_format): logits, end_points = ghost_net( inputs, dw_code=self.dw_code, ratio_code=self.ratio_code, se=self.se, num_classes=self.num_classes, dropout_keep_prob=self.dropout_keep_prob, min_depth=8, depth_multiplier=self.depth_multiplier, depth=self.depth, conv_defs=None, prediction_fn=tf.contrib.layers.softmax, spatial_squeeze=True, reuse=None, scope=self.scope, global_pool=False) return logits
def backbone_scope(freeze): """ Args: freeze (bool): whether to freeze all the variables under the scope """ def nonlin(x): x = get_norm()(x) return tf.nn.relu(x) with argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'), \ argscope(Conv2D, use_bias=False, activation=nonlin, kernel_initializer=tf.variance_scaling_initializer( scale=2.0, mode='fan_out')), \ ExitStack() as stack: if cfg.BACKBONE.NORM in ['FreezeBN', 'SyncBN']: if freeze or cfg.BACKBONE.NORM == 'FreezeBN': stack.enter_context(argscope(BatchNorm, training=False)) else: stack.enter_context( argscope(BatchNorm, sync_statistics='nccl' if cfg.TRAINER == 'replicated' else 'horovod')) if freeze: stack.enter_context( freeze_variables(stop_gradient=False, skip_collection=True)) else: # the layers are not completely freezed, but we may want to only freeze the affine if cfg.BACKBONE.FREEZE_AFFINE: stack.enter_context(custom_getter_scope(freeze_affine_getter)) yield
def create_resnet_model(image, is_training, isMSC=False, isASPP=False): mode = 'resnet' bottleneck = { 'resnet': resnet_bottleneck, 'preact': preresnet_bottleneck, 'se': se_resnet_bottleneck }[mode] basicblock = preresnet_basicblock if mode == 'preact' else resnet_basicblock num_blocks, block_func = { 18: ([2, 2, 2, 2], basicblock), 34: ([3, 4, 6, 3], basicblock), 50: ([3, 4, 6, 3], bottleneck), 101: ([3, 4, 23, 3], bottleneck), 152: ([3, 8, 36, 3], bottleneck) }[101] with argscope([Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm], data_format='NHWC'): with argscope([BatchNorm], use_local_stat=False): logits = resnet_backbone(image, num_blocks, resnet_group, resnet_group_dilation, block_func, resnet_bottleneck_dilation) # image075 = tf.image.resize_images(image, [int(image.get_shape().as_list()[1]*0.75), int(image.get_shape().as_list()[2]*0.75)]) # with tf.variable_scope('', reuse=True): # logits075 = resnet_backbone(image075, num_blocks, resnet_group, resnet_group_dilation, block_func, resnet_bottleneck_dilation) # image05 = tf.image.resize_images(image, [int(image.get_shape().as_list()[1]*0.5), int(image.get_shape().as_list()[2]*0.5)]) # with tf.variable_scope('', reuse=True): # logits05 = resnet_backbone(image05, num_blocks, resnet_group, resnet_group_dilation, block_func, resnet_bottleneck_dilation) # logits = tf.reduce_max(tf.stack([logits100, tf.image.resize_images(logits075, tf.shape(logits100)[1:3, ]), tf.image.resize_images(logits05, tf.shape(logits100)[1:3, ])]), axis=0) # with argscope([Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm], data_format='NHWC'): # with argscope([BatchNorm], use_local_stat=False): # logits = resnet_backbone(image, num_blocks, resnet_group, resnet_group_dilation, block_func, resnet_bottleneck_dilation) return logits
def backbone_argscope(): def nonlin(x): x = get_norm()(x) return tf.nn.relu(x) with argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'), \ argscope(Conv2D, use_bias=False, activation=nonlin), \ argscope(BatchNorm, training=False), \ custom_getter_scope(maybe_freeze_affine): yield
def get_logits(self, image): constant_init = tf.constant_initializer(1) with argscope([mpusim_conv2d, mpusim_separable_convolution2d], data_format=self.data_format, activations_datatype_size_byte=self.activations_datatype_size_byte, weights_datatype_size_byte=self.weights_datatype_size_byte, results_datatype_size_byte=self.results_datatype_size_byte, systolic_array_height=self.systolic_array_height, systolic_array_width=self.systolic_array_width, activation_fifo_depth=8, accumulator_array_height=self.accumulator_array_height, log_file_output_dir=self.mpusim_logdir, model_name='mobilenet_v3_sys_arr_h_{}_sys_arr_w_{}'.format(self.systolic_array_height, self.systolic_array_width)), \ argscope([mpusim_conv2d], activation=tf.nn.relu, kernel_initializer=constant_init): l = mpusim_conv2d('Conv', image, 16, 3, strides=(2, 2), activation=hard_swish) l = mbv3_op_se(l, ef=1, n=16, k=3, s=2) l = mbv3_op(l, ef=72. / 16, n=24, k=3, s=2) l = mbv3_op(l, ef=(88. / 24), n=24, k=3, s=1) l = mbv3_op_se(l, ef=4, n=40, k=5, s=2, act=hard_swish) l = mbv3_op_se(l, ef=6, n=40, k=5, s=1, act=hard_swish) l = mbv3_op_se(l, ef=6, n=40, k=5, s=1, act=hard_swish) l = mbv3_op_se(l, ef=3, n=48, k=5, s=1, act=hard_swish) l = mbv3_op_se(l, ef=3, n=48, k=5, s=1, act=hard_swish) l = mbv3_op_se(l, ef=6, n=96, k=5, s=2, act=hard_swish) l = mbv3_op_se(l, ef=6, n=96, k=5, s=1, act=hard_swish) l = mbv3_op_se(l, ef=6, n=96, k=5, s=1, act=hard_swish) l = mpusim_conv2d('Conv_1', l, 576, 1, activation=hard_swish) l = reduce_to_1x1(l, default_size=7, stride=1, padding='VALID') l = mpusim_conv2d('Conv_2', l, 1024, 1, activation=hard_swish) l = mpusim_conv2d( 'Conv2d_1c_1x1', l, 1000, 1, activation=None, bias_initializer=tf.compat.v1.zeros_initializer()) return tf.squeeze(l, [1, 2])
def get_logits(self, image, scale): update_basis = True if self.fixed_qa and scale != max(self.scales): update_basis = False with argscope([Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format=self.data_format), \ argscope([QuantizedActiv], nbit=self.qa, update_basis=update_basis): if self.mode == 'preact': group_func = preresnet_group elif self.mode == 'preact_typeA': group_func = preresnet_group_typeA else: group_func = resnet_group return resnet_backbone(image, self.num_blocks, group_func, self.block_func, self.qw, scale)
def get_logits(self, image): constant_init = tf.constant_initializer(1) with argscope([mpusim_conv2d, MaxPooling, BatchNorm], data_format='NHWC'), \ argscope([mpusim_conv2d, mpusim_fully_connected], kernel_initializer=constant_init, activations_datatype_size_byte=self.activations_datatype_size_byte, weights_datatype_size_byte=self.weights_datatype_size_byte, results_datatype_size_byte=self.results_datatype_size_byte, systolic_array_height=self.systolic_array_height, systolic_array_width=self.systolic_array_width, activation_fifo_depth=8, accumulator_array_height=self.accumulator_array_height, log_file_output_dir=self.mpusim_logdir, model_name='vgg16_sys_arr_h_{}_sys_arr_w_{}_acc_arr_h_{}'.format(self.systolic_array_height, self.systolic_array_width, self.accumulator_array_height)): logits = ( LinearWrap(image).apply(convnormrelu, 'conv1_1', 64).apply(convnormrelu, 'conv1_2', 64).MaxPooling('pool1', 2) # 112 .apply(convnormrelu, 'conv2_1', 128).apply(convnormrelu, 'conv2_2', 128).MaxPooling('pool2', 2) # 56 .apply(convnormrelu, 'conv3_1', 256).apply(convnormrelu, 'conv3_2', 256).apply(convnormrelu, 'conv3_3', 256).MaxPooling('pool3', 2) # 28 .apply(convnormrelu, 'conv4_1', 512).apply(convnormrelu, 'conv4_2', 512).apply(convnormrelu, 'conv4_3', 512).MaxPooling('pool4', 2) # 14 .apply(convnormrelu, 'conv5_1', 512).apply(convnormrelu, 'conv5_2', 512).apply(convnormrelu, 'conv5_3', 512).MaxPooling('pool5', 2) ## 7 .mpusim_fully_connected( 'fc6', 4096).tf.nn.relu(name='fc6_relu').mpusim_fully_connected( 'fc7', 4096).tf.nn.relu( name='fc7_relu').mpusim_fully_connected( 'fc8', 1000)()) add_param_summary(('.*', ['histogram', 'rms'])) return logits
def get_logits(self, image): with argscope([Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format='channels_first'), \ argscope(Conv2D, use_bias=False): group = args.group if not args.v2: # Copied from the paper channels = { 3: [240, 480, 960], 4: [272, 544, 1088], 8: [384, 768, 1536] } mul = group * 4 # #chan has to be a multiple of this number channels = [ int(math.ceil(x * args.ratio / mul) * mul) for x in channels[group] ] # The first channel must be a multiple of group first_chan = int(math.ceil(24 * args.ratio / group) * group) else: # Copied from the paper channels = { 0.5: [48, 96, 192], 1.: [116, 232, 464] }[args.ratio] first_chan = 24 logger.info("#Channels: " + str([first_chan] + channels)) l = Conv2D('conv1', image, first_chan, 3, strides=2, activation=BNReLU) l = MaxPooling('pool1', l, 3, 2, padding='SAME') l = shufflenet_stage('stage2', l, channels[0], 4, group) l = shufflenet_stage('stage3', l, channels[1], 8, group) l = shufflenet_stage('stage4', l, channels[2], 4, group) if args.v2: l = Conv2D('conv5', l, 1024, 1, activation=BNReLU) l = GlobalAvgPooling('gap', l) logits = FullyConnected('linear', l, 1000) return logits
def get_logits(self, image): def shufflenet_unit(l, out_channel, group, stride): in_shape = l.get_shape().as_list() in_channel = in_shape[1] shortcut = l # We do not apply group convolution on the first pointwise layer # because the number of input channels is relatively small. first_split = group if in_channel != 16 else 1 l = Conv2D('conv1', l, out_channel // 4, 1, split=first_split, nl=BNReLU) l = channel_shuffle(l, group) l = DepthConv('dconv', l, out_channel // 4, 3, nl=BN, stride=stride) l = Conv2D('conv2', l, out_channel if stride == 1 else out_channel - in_channel, 1, split=group, nl=BN) if stride == 1: # unit (b) output = tf.nn.relu(shortcut + l) else: # unit (c) shortcut = AvgPooling('avgpool', shortcut, 3, 2, padding='SAME') output = tf.concat([shortcut, tf.nn.relu(l)], axis=1) return output with argscope([Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format=self.data_format), \ argscope(Conv2D, use_bias=False): group = 8 channels = [224, 416, 832] l = Conv2D('conv1', image, 16, 3, stride=2, nl=BNReLU) l = MaxPooling('pool1', l, 3, 2, padding='SAME') with tf.variable_scope('group1'): for i in range(4): with tf.variable_scope('block{}'.format(i)): l = shufflenet_unit(l, channels[0], group, 2 if i == 0 else 1) with tf.variable_scope('group2'): for i in range(6): with tf.variable_scope('block{}'.format(i)): l = shufflenet_unit(l, channels[1], group, 2 if i == 0 else 1) with tf.variable_scope('group3'): for i in range(4): with tf.variable_scope('block{}'.format(i)): l = shufflenet_unit(l, channels[2], group, 2 if i == 0 else 1) l = GlobalAvgPooling('gap', l) logits = FullyConnected('linear', l, 1000) return logits
def get_logits(self, image): with argscope([Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm], data_format=self.data_format): return resnet_backbone( image, self.num_blocks, preresnet_group if self.mode == 'preact' else resnet_group, self.block_func)
def pan_model(features): """ Args: features ([tf.Tensor]): ResNet features c2-c5 Returns: [tf.Tensor]: FPN features p2-p6 """ num_channel = 256 use_gn = config.NORM == 'GN' with argscope(Conv2D, data_format='channels_first', activation=tf.identity, use_bias=True, kernel_initializer=tf.variance_scaling_initializer(scale=1.)): all_p = features pan_lat_sum_654321 = [] for idx, lat in enumerate(all_p[::-1]): if idx == 0: pan_lat_sum_654321.append(lat) else: #lat = lat + tf.transpose(tf.image.resize_nearest_neighbor(tf.transpose(lat_sum_5432[-1], [0, 2, 3, 1]), size=tf.shape(lat)[-2:]), [0, 3, 1, 2]) lat = tf.pad(pan_lat_sum_654321[-1], [[0, 0], [0, 0], [0, 1], [0, 1]]) lat = Conv2D('pan_down_{}'.format(6-idx), lat, 256, 3, stride=2, padding='VALID') #lat = lat + upsample2x('upsample_lat{}'.format(6 - idx), lat_sum_5432[-1]) pan_lat_sum_654321.append(lat) pan_654321 = [Conv2D('panhoc_3x3_p{}'.format(i + 2), c, num_channel, 3) for i, c in enumerate(pan_lat_sum_654321[::-1])] return pan_654321
def get_logits(self, x): with argscope( [Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format='channels_first'): x = res_init_block(x=x, in_channels=self.in_channels, out_channels=self.init_block_channels, name="features/init_block") in_channels = self.init_block_channels for i, channels_per_stage in enumerate(self.channels): for j, out_channels in enumerate(channels_per_stage): strides = 2 if (j == 0) and (i != 0) else 1 x = res_unit("features/stage{}/unit{}".format( i + 1, j + 1), x, in_channels=in_channels, out_channels=out_channels, strides=strides, bottleneck=self.bottleneck, conv1_stride=self.conv1_stride, use_se=self.use_se) in_channels = out_channels # x = AvgPooling( # "final_pool", # x, # pool_size=7, # strides=1) x = GlobalAvgPooling("features/final_pool", x) x = FullyConnected("output", x, units=self.classes) return x
def atrous_spatial_pyramid_pooling(logits): # Compute the ASPP. logits_size = tf.shape(logits)[1:3] with argscope(Conv2D, filters=256, kernel_size=3, activation=BNReLU): ASPP_1 = Conv2D('aspp_conv1', logits, kernel_size=1) ASPP_2 = Conv2D('aspp_conv2', logits, dilation_rate=cfg.atrous_rates[0]) ASPP_3 = Conv2D('aspp_conv3', logits, dilation_rate=cfg.atrous_rates[1]) ASPP_4 = Conv2D('aspp_conv4', logits, dilation_rate=cfg.atrous_rates[2]) # ImagePooling = GlobalAvgPooling('image_pooling', logits) ImagePooling = tf.reduce_mean(logits, [1, 2], name='global_average_pooling', keepdims=True) image_level_features = Conv2D('image_level_conv', ImagePooling, kernel_size=1) image_level_features = tf.image.resize_bilinear(image_level_features, logits_size, name='upsample') output = tf.concat([ASPP_1, ASPP_2, ASPP_3, ASPP_4, image_level_features], -1, name='concat') output = Conv2D('conv_after_concat', output, 256, 1, activation=BNReLU) return output
def get_logits(self, image): with argscope(Conv2D, kernel_shape=3, nl=tf.nn.relu): l = Conv2D('conv1_1', image, 64) l = Conv2D('conv1_2', l, 64) l = MaxPooling('pool1', l, 2) l = Conv2D('conv2_1', l, 128) l = Conv2D('conv2_2', l, 128) l = MaxPooling('pool2', l, 2) l = Conv2D('conv3_1', l, 256) l = Conv2D('conv3_2', l, 256) l = Conv2D('conv3_3', l, 256) l = MaxPooling('pool3', l, 2) l = Conv2D('conv4_1', l, 512) l = Conv2D('conv4_2', l, 512) l = Conv2D('conv4_3', l, 512) l = MaxPooling('pool4', l, 2) l = Conv2D('conv5_1', l, 512) l = Conv2D('conv5_2', l, 512) l = Conv2D('conv5_3', l, 512) l = MaxPooling('pool5', l, 2) l = FullyConnected('linear', l, 4096, nl=tf.nn.relu) l = Dropout("dropout1",l , 0.5) l = FullyConnected('linear2', l, 4096, nl=tf.nn.relu) l = Dropout("dropout2", l, 0.5) l = FullyConnected('linear3', l, 1000, nl=tf.identity) return l
def maybe_syncbn_scope(): if cfg.BACKBONE.NORM == 'SyncBN': assert cfg.BACKBONE.FREEZE_AT == 2 # TODO add better support with argscope(BatchNorm, training=None, sync_statistics='nccl'): yield else: yield
def build_graph(self, image, label, indices): """ The default tower function. """ image = self.image_preprocess(image) assert self.data_format == 'NCHW' image = tf.transpose(image, [0, 3, 1, 2]) with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE): # BatchNorm always comes with trouble. We use the testing mode of it during attack. with freeze_collection([tf.GraphKeys.UPDATE_OPS]), argscope(BatchNorm, training=False): image, target_label = self.attacker.attack(image, label, self.get_logits) image = tf.stop_gradient(image, name='adv_training_sample') logits = self.get_logits(image) loss = ImageNetModel.compute_loss_and_error( logits, label, label_smoothing=self.label_smoothing) AdvImageNetModel.compute_attack_success(logits, target_label) if not self.training: return wd_loss = regularize_cost(self.weight_decay_pattern, tf.contrib.layers.l2_regularizer(self.weight_decay), name='l2_regularize_loss') add_moving_summary(loss, wd_loss) total_cost = tf.add_n([loss, wd_loss], name='cost') if self.loss_scale != 1.: logger.info("Scaling the total loss by {} ...".format(self.loss_scale)) return total_cost * self.loss_scale else: return total_cost
def get_logits(self, image): tf.summary.image('input-image', image, max_outputs=3) with argscope( Conv2D, kernel_shape=3, nl=tf.nn.relu, kernel_initializer=tf.contrib.layers.xavier_initializer()): logits = ( LinearWrap(image).Conv2D('conv1_1', 64).Conv2D('conv1_2', 64).MaxPooling('pool1', 2) # 112 .Conv2D('conv2_1', 128).Conv2D('conv2_2', 128).MaxPooling('pool2', 2) # 56 .Conv2D('conv3_1', 256).Conv2D('conv3_2', 256).Conv2D( 'conv3_3', 256).Conv2D('conv3_4', 256).MaxPooling('pool3', 2) # 28 .Conv2D('conv4_1', 512).Conv2D('conv4_2', 512).Conv2D( 'conv4_3', 512).Conv2D('conv4_4', 512).MaxPooling('pool4', 2) # 14 .Conv2D('conv5_1', 512).Conv2D('conv5_2', 512).Conv2D( 'conv5_3', 512).Conv2D('conv5_4', 512).MaxPooling('pool5', 2) # 7 .FullyConnected('fc6', 4096, nl=tf.nn.relu).Dropout( 'drop0', 0.5).FullyConnected('fc7', 4096, nl=tf.nn.relu).Dropout( 'drop1', 0.5).FullyConnected('fc8', out_dim=1000, nl=tf.identity)()) return logits
def get_logits(self, image): with argscope(Conv2D, kernel_size=3, kernel_initializer=tf.variance_scaling_initializer(scale=2.)), \ argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'): logits = ( LinearWrap(image).apply(convnormrelu, 'conv1_1', 64).apply(convnormrelu, 'conv1_2', 64).MaxPooling('pool1', 2) # 112 .apply(convnormrelu, 'conv2_1', 128).apply(convnormrelu, 'conv2_2', 128).MaxPooling('pool2', 2) # 56 .apply(convnormrelu, 'conv3_1', 256).apply(convnormrelu, 'conv3_2', 256).apply(convnormrelu, 'conv3_3', 256).MaxPooling('pool3', 2) # 28 .apply(convnormrelu, 'conv4_1', 512).apply(convnormrelu, 'conv4_2', 512).apply(convnormrelu, 'conv4_3', 512).MaxPooling('pool4', 2) # 14 .apply(convnormrelu, 'conv5_1', 512).apply(convnormrelu, 'conv5_2', 512).apply(convnormrelu, 'conv5_3', 512).MaxPooling('pool5', 2) # 7 .FullyConnected( 'fc6', 4096, kernel_initializer=tf.random_normal_initializer( stddev=0.001)).tf.nn.relu(name='fc6_relu'). Dropout('drop0', rate=0.5).FullyConnected( 'fc7', 4096, kernel_initializer=tf.random_normal_initializer( stddev=0.001)).tf.nn.relu(name='fc7_relu').Dropout( 'drop1', rate=0.5).FullyConnected( 'fc8', 1000, kernel_initializer=tf. random_normal_initializer(stddev=0.01))()) add_param_summary(('.*', ['histogram', 'rms'])) return logits
def residual(x, chan, kernel_shape=3): with argscope([Conv3D], nl=INELU, stride=1, kernel_shape=kernel_shape): input = x return (LinearWrap(x) .Conv3D('conv0', chan, padding='SAME', kernel_shape=(1,3,3)) .Conv3D('conv1', chan/2, padding='SAME') .Conv3D('conv2', chan, padding='SAME', nl=tf.identity) # .InstanceNorm('inorm') ()) + input
def vgg16_fc(image): with argscope(Conv2D, kernel_initializer=tf.variance_scaling_initializer(scale=2.)), \ argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'): x = FullyConnected( 'fc6', image, 4096, kernel_initializer=tf.random_normal_initializer(stddev=0.001), activation=tf.nn.relu) x = Dropout('drop0', x, rate=0.5) x = FullyConnected( 'fc7', x, 4096, kernel_initializer=tf.random_normal_initializer(stddev=0.001), activation=tf.nn.relu) x = Dropout('drop1', x, rate=0.5) return x
def build_graph(self, query, key): # setup queue queue_init = tf.math.l2_normalize(tf.random.normal( [self.queue_size, self.feature_dim]), axis=1) queue = tf.get_variable('queue', initializer=queue_init, trainable=False) queue_ptr = tf.get_variable('queue_ptr', [], initializer=tf.zeros_initializer(), dtype=tf.int64, trainable=False) tf.add_to_collection(tf.GraphKeys.MODEL_VARIABLES, queue) tf.add_to_collection(tf.GraphKeys.MODEL_VARIABLES, queue_ptr) # query encoder q_feat = self.net.forward(query) # NxC q_feat = tf.math.l2_normalize(q_feat, axis=1) # key encoder shuffled_key, shuffle_idxs = batch_shuffle(key) shuffled_key.set_shape([self.batch_size, None, None, None]) with tf.variable_scope("momentum_encoder"), \ varreplace.freeze_variables(skip_collection=True), \ argscope(BatchNorm, ema_update='skip'): # don't maintain EMA (will not be used at all) key_feat = xla.compile(lambda: self.net.forward(shuffled_key))[0] # key_feat = self.net.forward(shuffled_key) key_feat = tf.math.l2_normalize(key_feat, axis=1) # NxC key_feat = batch_unshuffle(key_feat, shuffle_idxs) key_feat = tf.stop_gradient(key_feat) # loss l_pos = tf.reshape(tf.einsum('nc,nc->n', q_feat, key_feat), (-1, 1)) # nx1 l_neg = tf.einsum('nc,kc->nk', q_feat, queue) # nxK logits = tf.concat([l_pos, l_neg], axis=1) # nx(1+k) logits = logits * (1 / self.temp) labels = tf.zeros(self.batch_size, dtype=tf.int64) # n loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels) loss = tf.reduce_mean(loss, name='xentropy-loss') acc = tf.reduce_mean(tf.cast( tf.equal(tf.math.argmax(logits, axis=1), labels), tf.float32), name='train-acc') # update queue (depend on l_neg) with tf.control_dependencies([l_neg]): queue_push_op = self.push_queue(queue, queue_ptr, key_feat) tf.add_to_collection(tf.GraphKeys.UPDATE_OPS, queue_push_op) wd_loss = regularize_cost(".*", l2_regularizer(1e-4), name='l2_regularize_loss') add_moving_summary(acc, loss, wd_loss) total_cost = tf.add_n([loss, wd_loss], name='cost') return total_cost
def residual_enc(x, chan, kernel_shape=3): with argscope([Conv3D, DeConv3D], nl=INELU, stride=1, kernel_shape=kernel_shape): x = (LinearWrap(x) # .Dropout('drop', 0.75) .Conv3D('conv_i', chan, stride=(1, 2, 2)) .residual('res_', chan, kernel_shape=kernel_shape) .Conv3D('conv_o', chan, stride=1, kernel_shape=(1,3,3)) ()) return x
def VGG16(image, classes=5): with argscope(Conv2D, kernel_initializer=tf.variance_scaling_initializer(scale=2.)), \ argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'): image_channel_first = tf.transpose(image, [0, 3, 1, 2]) latent = ( LinearWrap(image_channel_first).apply( convnormrelu, 'conv1_1', 64).apply(convnormrelu, 'conv1_2', 64).MaxPooling('pool1', 2) # 112 .apply(convnormrelu, 'conv2_1', 128).apply(convnormrelu, 'conv2_2', 128).MaxPooling('pool2', 2) # 56 .apply(convnormrelu, 'conv3_1', 256).apply(convnormrelu, 'conv3_2', 256).apply(convnormrelu, 'conv3_3', 256).MaxPooling('pool3', 2) # 28 .apply(convnormrelu, 'conv4_1', 512).apply(convnormrelu, 'conv4_2', 512).apply(convnormrelu, 'conv4_3', 512).MaxPooling('pool4', 2) # 14 .apply(convnormrelu, 'conv5_1', 512).apply(convnormrelu, 'conv5_2', 512).apply(convnormrelu, 'conv5_3', 512).MaxPooling('pool5', 2)()) # 7 logits = (LinearWrap(latent).FullyConnected( 'fc6', 4096, kernel_initializer=tf.random_normal_initializer(stddev=0.001) ).tf.nn.relu(name='fc6_relu').Dropout( 'drop0', rate=0.5).FullyConnected( 'fc7', 4096, kernel_initializer=tf.random_normal_initializer( stddev=0.001)).tf.nn.relu(name='fc7_relu').Dropout( 'drop1', rate=0.5).FullyConnected( 'fc8', classes, kernel_initializer=tf.random_normal_initializer( stddev=0.01))()) return logits, latent
def get_logits(self, image): constant_init = tf.constant_initializer(1) with argscope([mpusim_conv2d, MaxPooling, GlobalAvgPooling, BatchNorm], data_format='NHWC'), \ argscope([mpusim_conv2d, mpusim_fully_connected], kernel_initializer=constant_init, activations_datatype_size_byte=self.activations_datatype_size_byte, weights_datatype_size_byte=self.weights_datatype_size_byte, results_datatype_size_byte=self.results_datatype_size_byte, systolic_array_height=self.systolic_array_height, systolic_array_width=self.systolic_array_width, activation_fifo_depth=8, accumulator_array_height=self.accumulator_array_height, log_file_output_dir=self.mpusim_logdir, model_name='resnext_{}_sys_arr_w_{}_acc_arr_h_{}'.format(self.resnet_depth, self.systolic_array_height, self.systolic_array_width)): return resnet_backbone(image, self.num_blocks, resnet_group, self.block_func)
def get_logits(self, image): group_func = resnet_group block_func = resnet_bottleneck num_blocks = [3, 4, 6, 3] with argscope( [Conv2D, MaxPooling, GlobalAvgPooling, BatchNorm], data_format='NCHW'), \ argscope(Conv2D, nl=tf.identity, use_bias=False, W_init=tf.variance_scaling_initializer(scale=2.0, mode='fan_out')): logits = (LinearWrap(image) .Conv2D('conv0', 64, 7, stride=2, nl=BNReLU) .MaxPooling('pool0', shape=3, stride=2, padding='SAME') .apply(group_func, 'group0', block_func, 64, num_blocks[0], 1) .apply(group_func, 'group1', block_func, 128, num_blocks[1], 2) .apply(group_func, 'group2', block_func, 256, num_blocks[2], 2) .apply(group_func, 'group3', block_func, 512, num_blocks[3], 2) .GlobalAvgPooling('gap') .FullyConnected('linear', 1000, nl=tf.identity)()) return logits
def get_logits(self, image): with argscope([Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format='channels_first'), \ argscope(Conv2D, use_bias=False): group = args.group if not args.v2: # Copied from the paper channels = { 3: [240, 480, 960], 4: [272, 544, 1088], 8: [384, 768, 1536] } mul = group * 4 # #chan has to be a multiple of this number channels = [int(math.ceil(x * args.ratio / mul) * mul) for x in channels[group]] # The first channel must be a multiple of group first_chan = int(math.ceil(24 * args.ratio / group) * group) else: # Copied from the paper channels = { 0.5: [48, 96, 192], 1.: [116, 232, 464] }[args.ratio] first_chan = 24 logger.info("#Channels: " + str([first_chan] + channels)) l = Conv2D('conv1', image, first_chan, 3, strides=2, activation=BNReLU) l = MaxPooling('pool1', l, 3, 2, padding='SAME') l = shufflenet_stage('stage2', l, channels[0], 4, group) l = shufflenet_stage('stage3', l, channels[1], 8, group) l = shufflenet_stage('stage4', l, channels[2], 4, group) if args.v2: l = Conv2D('conv5', l, 1024, 1, activation=BNReLU) l = GlobalAvgPooling('gap', l) logits = FullyConnected('linear', l, 1000) return logits
def get_logits(self, image): with argscope([Conv2D, MaxPooling, AvgPooling, GlobalAvgPooling, BatchNorm], data_format=self.data_format), \ argscope([QuantizedActiv], nbit=self.qa): if self.mode == 'vgg': return vgg_backbone(image, self.qw) elif self.mode == 'alexnet': return alexnet_backbone(image, self.qw) elif self.mode == 'googlenet': return googlenet_backbone(image, self.qw) elif self.mode == 'densenet': return densenet_backbone(image, self.qw) else: if self.mode == 'preact': group_func = preresnet_group elif self.mode == 'preact_typeA': group_func = preresnet_group_typeA else: group_func = resnet_group return resnet_backbone( image, self.num_blocks, group_func, self.block_func, self.qw)
def get_logits(self, image): with argscope(Conv2D, kernel_initializer=tf.variance_scaling_initializer(scale=2.)), \ argscope([Conv2D, MaxPooling, BatchNorm], data_format='channels_first'): logits = (LinearWrap(image) .apply(convnormrelu, 'conv1_1', 64) .apply(convnormrelu, 'conv1_2', 64) .MaxPooling('pool1', 2) # 112 .apply(convnormrelu, 'conv2_1', 128) .apply(convnormrelu, 'conv2_2', 128) .MaxPooling('pool2', 2) # 56 .apply(convnormrelu, 'conv3_1', 256) .apply(convnormrelu, 'conv3_2', 256) .apply(convnormrelu, 'conv3_3', 256) .MaxPooling('pool3', 2) # 28 .apply(convnormrelu, 'conv4_1', 512) .apply(convnormrelu, 'conv4_2', 512) .apply(convnormrelu, 'conv4_3', 512) .MaxPooling('pool4', 2) # 14 .apply(convnormrelu, 'conv5_1', 512) .apply(convnormrelu, 'conv5_2', 512) .apply(convnormrelu, 'conv5_3', 512) .MaxPooling('pool5', 2) # 7 .FullyConnected('fc6', 4096, kernel_initializer=tf.random_normal_initializer(stddev=0.001)) .tf.nn.relu(name='fc6_relu') .Dropout('drop0', rate=0.5) .FullyConnected('fc7', 4096, kernel_initializer=tf.random_normal_initializer(stddev=0.001)) .tf.nn.relu(name='fc7_relu') .Dropout('drop1', rate=0.5) .FullyConnected('fc8', 1000, kernel_initializer=tf.random_normal_initializer(stddev=0.01))()) add_param_summary(('.*', ['histogram', 'rms'])) return logits