def resnet50(images, args, trainable=True, training=True): weight_regularizer = flow.regularizers.l2(args.wd) if args.wd > 0.0 and args.wd < 1.0 else None builder = ResnetBuilder(weight_regularizer, trainable, training, args.channel_last, args.fuse_bn_relu, args.fuse_bn_add_relu) if args.pad_output: if args.channel_last: paddings = ((0, 0), (0, 0), (0, 0), (0, 1)) else: paddings = ((0, 0), (0, 1), (0, 0), (0, 0)) images = flow.pad(images, paddings=paddings) with flow.scope.namespace("Resnet"): stem = builder.resnet_stem(images) body = builder.resnet_conv_x_body(stem) pool5 = flow.nn.avg_pool2d( body, ksize=7, strides=1, padding="VALID", data_format=builder.data_format, name="pool5", ) fc1001 = flow.layers.dense( flow.reshape(pool5, (pool5.shape[0], -1)), units=1000, use_bias=True, kernel_initializer=flow.variance_scaling_initializer(2, 'fan_in', 'random_normal'), bias_initializer=flow.zeros_initializer(), kernel_regularizer=weight_regularizer, bias_regularizer=weight_regularizer, trainable=trainable, name="fc1001", ) return fc1001
def _conv2d( name, input, filters, kernel_size, strides=1, padding="SAME", data_format="NCHW", dilations=1, trainable=True, # weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"), weight_initializer=flow.variance_scaling_initializer( 2, "fan_in", "random_normal", data_format="NCHW"), weight_regularizer=flow.regularizers.l2(1.0 / 32768), ): weight = flow.get_variable( name + "-weight", shape=(filters, input.shape[1], kernel_size, kernel_size), dtype=input.dtype, initializer=weight_initializer, regularizer=weight_regularizer, model_name="weight", trainable=trainable, ) return flow.nn.conv2d(input, weight, strides, padding, data_format, dilations, name=name)
def _conv2d( name, input, filters, kernel_size, strides=1, padding="SAME", data_format="NCHW", dilations=1, trainable=True, weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"), ): weight = flow.get_variable( name + "-weight", shape=(filters, input.shape[1], kernel_size, kernel_size), dtype=input.dtype, initializer=weight_initializer, trainable=trainable, ) return flow.nn.conv2d(input, weight, strides, padding, data_format, dilations, name=name)
def resnet50(images, trainable=True, need_transpose=False): # note: images.shape = (N C H W) in cc's new dataloader, transpose is not needed anymore if need_transpose: images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) with flow.scope.namespace("Resnet"): stem = resnet_stem(images) body = resnet_conv_x_body(stem, lambda x: x) pool5 = flow.nn.avg_pool2d( body, ksize=7, strides=1, padding="VALID", data_format="NCHW", name="pool5", ) fc1001 = flow.layers.dense( flow.reshape(pool5, (pool5.shape[0], -1)), units=1000, use_bias=True, kernel_initializer=flow.variance_scaling_initializer( 2, "fan_in", "random_normal"), # kernel_initializer=flow.xavier_uniform_initializer(), bias_initializer=flow.random_uniform_initializer(), kernel_regularizer=flow.regularizers.l2(1.0 / 32768), trainable=trainable, name="fc1001", ) return fc1001
def __init__(self, weight_regularizer, trainable=True, training=True, channel_last=False, fuse_bn_relu=True, fuse_bn_add_relu=True): self.data_format = "NHWC" if channel_last else "NCHW" self.weight_initializer = flow.variance_scaling_initializer(2, 'fan_in', 'random_normal', data_format=self.data_format) self.weight_regularizer = weight_regularizer self.trainable = trainable self.training = training self.fuse_bn_relu = fuse_bn_relu self.fuse_bn_add_relu = fuse_bn_add_relu
def conv2d_layer( name, input, filters, kernel_size=3, strides=1, padding="SAME", data_format="NCHW", dilation_rate=1, activation="Relu", use_bias=True, weight_initializer=flow.variance_scaling_initializer( 2, 'fan_out', 'random_normal', data_format="NCHW"), bias_initializer=flow.zeros_initializer(), weight_regularizer=_get_regularizer(), # weight_decay bias_regularizer=_get_regularizer(), bn=True, reuse=False): name_ = name if reuse == False else name + "_reuse" weight_shape = (filters, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable( name + "_weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, ) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilation_rate, name=name_) if use_bias: bias = flow.get_variable( name + "_bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, ) output = flow.nn.bias_add(output, bias, data_format) if activation is not None: if activation == "Relu": if bn: # use of_layers(layers) batch_norm output = layers.batch_norm(output, name + "_bn", reuse=reuse) output = flow.nn.relu(output) else: output = flow.nn.relu(output) else: raise NotImplementedError return output
def _get_initializer(model_name): if model_name == "weight": return flow.variance_scaling_initializer(2.0, mode="fan_out", distribution="random_normal", data_format="NCHW") elif model_name == "bias": return flow.zeros_initializer() elif model_name == "gamma": return flow.ones_initializer() elif model_name == "beta": return flow.zeros_initializer() elif model_name == "dense_weight": return flow.variance_scaling_initializer(1 / 3, mode="fan_in", distribution="random_uniform") elif model_name == "dense_bias": return flow.random_uniform_initializer(0, 0.01)
def __init__( self, weight_regularizer, trainable=True, training=True, channel_last=False ): self.data_format = "NHWC" if channel_last else "NCHW" self.weight_initializer = flow.variance_scaling_initializer( 2, "fan_in", "random_normal", data_format=self.data_format ) self.weight_regularizer = weight_regularizer self.trainable = trainable self.training = training
def conv2d_layer( name, input, filters, kernel_size=1, strides=1, padding="VALID", data_format="NCHW", dilation_rate=1, activation="Relu", use_bias=True, weight_initializer=flow.variance_scaling_initializer( 2, 'fan_out', 'random_normal', data_format="NCHW"), bias_initializer=flow.zeros_initializer(), bn=True, ): weight_shape = (filters, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable( name + "_weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, ) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilation_rate, name=name) if use_bias: bias = flow.get_variable( name + "_bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, ) output = flow.nn.bias_add(output, bias, data_format) if activation is not None: if activation == "Relu": if bn: output = _batch_norm(output, name + "_bn") # flow.watch(output) output = flow.nn.relu(output) else: output = flow.nn.relu(output) else: raise NotImplementedError return output
def loss_con(self, one_hot_labels, feature): branch = feature fc_part1 = flow.layers.dense( flow.reshape(branch, (branch.shape[0], -1)), units=8, # 车辆颜色类别8 use_bias=True, kernel_initializer=flow.variance_scaling_initializer( 2, 'fan_in', 'random_normal'), bias_initializer=flow.zeros_initializer(), name="fc1", ) loss_con = flow.nn.softmax_cross_entropy_with_logits( one_hot_labels, fc_part1, name="softmax_loss1") return loss_con
def inflate_linear(inputs, linear2d, time_dim): # weight3d=linear2d.weight.data # weight3d=np.tile(weight3d,(1,time_dim)) # weight3d=weight3d/time_dim # init=flow.constant_initializer(weight3d) init = flow.random_uniform_initializer(minval=0, maxval=0.5) kernel_initializer = flow.variance_scaling_initializer( 2, 'fan_in', 'random_normal') weight_regularizer = flow.regularizers.l2(1.0 / 32768) linear3d = flow.layers.dense(inputs, linear2d.out_features, use_bias=True, bias_initializer=kernel_initializer, kernel_regularizer=weight_regularizer, bias_regularizer=weight_regularizer, trainable=True) return linear3d
def resnet50( images, trainable=True, need_transpose=False, training=True, wd=1.0 / 32768, channel_last=False, ): weight_regularizer = flow.regularizers.l2(wd) if wd > 0.0 and wd < 1.0 else None builder = ResnetBuilder(weight_regularizer, trainable, training, channel_last) # note: images.shape = (N C H W) in cc's new dataloader, transpose is not needed anymore if need_transpose: images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2]) if channel_last: # if channel_last=True, then change mode from 'nchw' to 'nhwc' images = flow.transpose(images, name="transpose", perm=[0, 2, 3, 1]) with flow.scope.namespace("Resnet"): stem = builder.resnet_stem(images) body = builder.resnet_conv_x_body(stem) pool5 = flow.nn.avg_pool2d( body, ksize=7, strides=1, padding="VALID", data_format=builder.data_format, name="pool5", ) fc1001 = flow.layers.dense( flow.reshape(pool5, (pool5.shape[0], -1)), units=1000, use_bias=True, kernel_initializer=flow.variance_scaling_initializer( 2, "fan_in", "random_normal" ), bias_initializer=flow.zeros_initializer(), kernel_regularizer=weight_regularizer, bias_regularizer=weight_regularizer, trainable=trainable, name="fc1001", ) return fc1001
def _conv_block(in_blob, index, filters, conv_times, data_format="NCHW"): conv_block = [] conv_block.insert(0, in_blob) weight_initializer = flow.variance_scaling_initializer( 2, 'fan_out', 'random_normal', data_format=data_format) for i in range(conv_times): conv_i = conv2d_layer( name="conv{}".format(index), input=conv_block[i], filters=filters, kernel_size=3, strides=1, data_format=data_format, weight_initializer=weight_initializer, bn=True, ) conv_block.append(conv_i) index += 1 return conv_block
def conv2d_layer( name, input, out_channel, kernel_size=3, strides=1, padding="SAME", data_format="NCHW", dilation_rate=1, use_bias=True, weight_initializer=flow.variance_scaling_initializer( 2, "fan_out", "random_normal", data_format="NCHW"), bias_initializer=flow.zeros_initializer(), trainable=True, ): weight_shape = (out_channel, input.shape[1], kernel_size, kernel_size) weight = flow.get_variable( name + "_weight", shape=weight_shape, dtype=input.dtype, initializer=weight_initializer, trainable=trainable, ) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilation_rate, name=name) if use_bias: bias = flow.get_variable( name + "_bias", shape=(out_channel, ), dtype=input.dtype, initializer=bias_initializer, trainable=trainable, ) output = flow.nn.bias_add(output, bias, data_format) return output
def test_float_initializer(test_case): initializers = [ flow.random_normal_initializer(mean=3, stddev=4), flow.random_uniform_initializer(minval=-6, maxval=18), flow.truncated_normal_initializer(mean=-5, stddev=8), flow.xavier_uniform_initializer(data_format="NCHW"), flow.xavier_uniform_initializer(data_format="NHWC"), flow.xavier_normal_initializer(data_format="NCHW"), flow.xavier_normal_initializer(data_format="NHWC"), flow.constant_initializer(value=4), flow.ones_initializer(), flow.zeros_initializer(), ] kaiming_args = GenArgDict( OrderedDict( shape=[SHAPE], mode=["fan_in", "fan_out", "fan_avg"], distribution=["random_normal", "random_uniform"], data_format=["NCHW", "NHWC"], negative_slope=[0.5], )) vs_args = GenArgDict( OrderedDict( scale=[3.4], mode=["fan_in", "fan_out", "fan_avg"], distribution=[ "truncated_normal", "random_normal", "random_uniform" ], data_format=["NCHW", "NHWC"], )) for args in kaiming_args: initializers.append(flow.kaiming_initializer(**args)) for args in vs_args: initializers.append(flow.variance_scaling_initializer(**args)) for initializer in initializers: CompareTwoDistribution(test_case, flow.float32, initializer)
def _conv2d( name, input, filters, kernel_size, strides=1, padding="SAME", data_format="NCHW", dilations=1, use_bias=True, trainable=True, weight_initializer=flow.variance_scaling_initializer(data_format="NCHW"), bias_initializer=flow.zeros_initializer()): weight = flow.get_variable(name + "_weight", shape=(filters, input.shape[1], kernel_size, kernel_size), dtype=input.dtype, initializer=weight_initializer, trainable=trainable) output = flow.nn.conv2d(input, weight, strides, padding, data_format, dilations, name=name) if use_bias: bias = flow.get_variable( name + "_bias", shape=(filters, ), dtype=input.dtype, initializer=bias_initializer, ) output = flow.nn.bias_add(output, bias, data_format) return output
def build_network(self, inputs): # resnet2d = torchvision.models.resnet50(pretrained=True) global time resnet2d = getresnet.getResnet() conv2d = Conv2d(3, 64, [7, 7], [2, 2], [3, 3], dilation=[1, 1]) out = inflate.inflate_conv(inputs, conv2d, time_dim=1, times=time) time += 1 # inputs=inflate.inflate_batch_norm(inputs,resnet2d.bn1) out = nn.relu(out) out = inflate.inflate_pool(out, kernel_size=3, padding=1, stride=2, dilation=1, time_dim=1) out = self._inflate_reslayer(out, resnet2d[0], c3d_idx=self.c3d_idx[0], nonlocal_idx=self.nl_idx[0], nonlocal_channels=256) print("layer1finish") out=self._inflate_reslayer(out,resnet2d[1], c3d_idx=self.c3d_idx[1], \ nonlocal_idx=self.nl_idx[1], nonlocal_channels=512) print("layer2finish") out=self._inflate_reslayer(out,resnet2d[2], c3d_idx=self.c3d_idx[2], \ nonlocal_idx=self.nl_idx[2], nonlocal_channels=1024) print("layer3finish") out= self._inflate_reslayer(out,resnet2d[3], c3d_idx=self.c3d_idx[3], \ nonlocal_idx=self.nl_idx[3], nonlocal_channels=2048) print("layer4finish") b, c, t, h, w = out.shape out = flow.transpose(out, perm=[0, 2, 1, 3, 4]) out = flow.reshape(out, shape=[b * t, c, h, w]) out = nn.max_pool2d(input=out, ksize=out.shape[2:], strides=None, padding="VALID") out = flow.reshape(out, shape=[b, t, -1]) out = flow.math.reduce_mean(out, axis=1) f = flow.layers.batch_normalization(inputs=out, momentum=0.997, epsilon=1.001e-5, center=True, scale=True, trainable=True, name="Resnet503D_linear_bn" + str(time)) time += 1 kernel_initializer = flow.variance_scaling_initializer( 2, 'fan_in', 'random_normal') weight_regularizer = flow.regularizers.l2(1.0 / 32768) y = flow.layers.dense(out, self.num_classes, use_bias=True, bias_initializer=kernel_initializer, kernel_regularizer=weight_regularizer, bias_regularizer=weight_regularizer, trainable=True) return y, f
def _get_kernel_initializer(): return flow.variance_scaling_initializer(distribution="random_normal", data_format="NCHW")
def _get_initializer(): return flow.variance_scaling_initializer(2.0, "fan_out", "random_normal", "NCHW")
def build_network(self, inputs, num_classes=1000, width=1.0, dropout=0.2, trainable=True, wd=1.0 / 32768): kernel_initializer = flow.variance_scaling_initializer( 2, 'fan_in', 'random_normal') weight_regularizer = flow.regularizers.l2( wd) if wd > 0.0 and wd < 1.0 else None # building first layer output_channel = _make_divisible(16 * width, 4) output = conv2d_layer_with_bn( "conv_gn_0", inputs, output_channel, kernel_size=3, strides=2, #??? padding="same", activation=None, use_bias=False) output = flow.nn.relu(output) input_channel = output_channel sum = 1 model = GhostBottleneck() # building inverted residual blocks for cfg in self.cfgs: for k, exp_size, c, se_ratio, s in cfg: output_channel = _make_divisible(c * width, 4) hidden_channel = _make_divisible(exp_size * width, 4) # self, inputs, mid_chs, out_chs, dw_kernel_size=3, stride=1, se_ratio=0. output = model.build_network(output, input_channel, hidden_channel, output_channel, dw_kernel_size=k, stride=s, se_ratio=se_ratio, times=sum) input_channel = output_channel sum += 1 output_channel = _make_divisible(exp_size * width, 4) output = conv2d_layer_with_bn("conv_gn_1", output, output_channel, 1, activation=None) #input_channel = output_channel # building last several layers output_channel = 1280 output = AdaptiveAvgPool2d(output, 1, 1) output = conv2d_layer("conv_gn_2", output, output_channel, 1, 1, padding="valid", use_bias=True) output = flow.nn.relu(output) print("***1") print(output.shape) output = flow.reshape(output, (output.shape[0], -1)) print("***2") print(output.shape) if dropout > 0.: output = nn.dropout(output, rate=dropout) output = flow.layers.dense(output, num_classes, use_bias=True, kernel_initializer=kernel_initializer, bias_initializer=flow.zeros_initializer(), kernel_regularizer=weight_regularizer, bias_regularizer=weight_regularizer, trainable=trainable) print("***3") print(output.shape) return output
def getQNetParams(var_name_prefix: str = "QNet", is_train: bool = True): weight_init = flow.variance_scaling_initializer( scale=1.0, mode="fan_in", distribution="truncated_normal", data_format="NCHW") bias_init = flow.constant_initializer(value=0.0) conv_prefix = "_conv1" conv1_weight = flow.get_variable( var_name_prefix + conv_prefix + "_weight", shape=(32, 4, 3, 3), dtype=flow.float32, initializer=weight_init, trainable=is_train, ) conv1_bias = flow.get_variable( var_name_prefix + conv_prefix + "_bias", shape=(32, ), dtype=flow.float32, initializer=bias_init, trainable=is_train, ) conv_prefix = "_conv2" conv2_weight = flow.get_variable( var_name_prefix + conv_prefix + "_weight", shape=(32, 32, 3, 3), dtype=flow.float32, initializer=weight_init, trainable=is_train, ) conv2_bias = flow.get_variable( var_name_prefix + conv_prefix + "_bias", shape=(32, ), dtype=flow.float32, initializer=bias_init, trainable=is_train, ) fc_prefix = "_fc1" fc1_weight = flow.get_variable( var_name_prefix + fc_prefix + "_weight", shape=(512, 32 * 16 * 16), dtype=flow.float32, initializer=weight_init, trainable=is_train, ) fc1_bias = flow.get_variable( var_name_prefix + fc_prefix + "_bias", shape=(512, ), dtype=flow.float32, initializer=bias_init, trainable=is_train, ) fc_prefix = "_fc2" fc2_weight = flow.get_variable( var_name_prefix + fc_prefix + "_weight", shape=(2, 512), dtype=flow.float32, initializer=weight_init, trainable=is_train, ) fc2_bias = flow.get_variable( var_name_prefix + fc_prefix + "_bias", shape=(2, ), dtype=flow.float32, initializer=bias_init, trainable=is_train, ) return ( conv1_weight, conv1_bias, conv2_weight, conv2_bias, fc1_weight, fc1_bias, fc2_weight, fc2_bias, )