def __init__(self, input_dims, num_classes, drop_out=0.1, name_scope=''): super(AuxHead, self).__init__(name_scope) learning_rate = 10.0 self.conv_1 = fluid.dygraph.Conv2D(input_dims, 256, filter_size=3, padding=1, bias_attr=False, param_attr=fluid.ParamAttr( learning_rate=learning_rate, trainable=True)) self.bn = nn.SyncBatchNorm( 256, weight_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), bias_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self.dropout = fluid.dygraph.Dropout(p=drop_out) self.conv_2 = fluid.dygraph.Conv2D( 256, num_classes, filter_size=1, param_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), bias_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True))
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, num_groups=1, act='relu', conv_lr=0.1, conv_decay=0., norm_decay=0., norm_type='bn', name=None): super(ConvBNLayer, self).__init__() self.act = act self._conv = nn.Conv2D( in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, groups=num_groups, weight_attr=ParamAttr( learning_rate=conv_lr, initializer=KaimingNormal()), bias_attr=False) if norm_type == 'sync_bn': self._batch_norm = nn.SyncBatchNorm(out_channels) else: self._batch_norm = nn.BatchNorm( out_channels, act=None, use_global_stats=False)
def SyncBatchNorm(*args, **kwargs): """In cpu environment nn.SyncBatchNorm does not have kernel so use nn.BatchNorm2D instead""" if paddle.get_device() == 'cpu' or os.environ.get( 'PADDLESEG_EXPORT_STAGE'): return nn.BatchNorm2D(*args, **kwargs) else: return nn.SyncBatchNorm(*args, **kwargs)
def __init__(self, in_c, out_c, filter_size, stride, padding, num_groups=1, act=None, lr_mult=1., conv_decay=0., norm_type='bn', norm_decay=0., freeze_norm=False, name=""): super(ConvBNLayer, self).__init__() self.act = act self.conv = nn.Conv2D( in_channels=in_c, out_channels=out_c, kernel_size=filter_size, stride=stride, padding=padding, groups=num_groups, weight_attr=ParamAttr( learning_rate=lr_mult, regularizer=L2Decay(conv_decay), name=name + "_weights"), bias_attr=False) norm_lr = 0. if freeze_norm else lr_mult param_attr = ParamAttr( learning_rate=norm_lr, regularizer=L2Decay(norm_decay), name=name + "_bn_scale", trainable=False if freeze_norm else True) bias_attr = ParamAttr( learning_rate=norm_lr, regularizer=L2Decay(norm_decay), name=name + "_bn_offset", trainable=False if freeze_norm else True) global_stats = True if freeze_norm else False if norm_type == 'sync_bn': self.bn = nn.SyncBatchNorm( out_c, weight_attr=param_attr, bias_attr=bias_attr) else: self.bn = nn.BatchNorm( out_c, act=None, param_attr=param_attr, bias_attr=bias_attr, use_global_stats=global_stats, moving_mean_name=name + '_bn_mean', moving_variance_name=name + '_bn_variance') norm_params = self.bn.parameters() if freeze_norm: for param in norm_params: param.stop_gradient = True
def __init__(self, dim_in, proj_dim=256, proj='convmlp'): super(ProjectionHead, self).__init__() if proj == 'linear': self.proj = nn.Conv2d(dim_in, proj_dim, kernel_size=1) elif proj == 'convmlp': self.proj = nn.Sequential( nn.Conv2d(dim_in, dim_in, kernel_size=1), nn.SyncBatchNorm(dim_in), nn.ReLU(), nn.Conv2d(dim_in, proj_dim, kernel_size=1))
def __init__(self): super(ModelConv, self).__init__() with supernet( kernel_size=(3, 4, 5, 7), channel=((4, 8, 12), (8, 12, 16), (8, 12, 16), (8, 12, 16))) as ofa_super: models = [] models += [nn.Conv2D(3, 4, 3, padding=1)] models += [nn.InstanceNorm2D(4)] models += [nn.SyncBatchNorm(4)] models += [ReLU()] models += [nn.Conv2D(4, 4, 3, groups=4)] models += [nn.InstanceNorm2D(4)] models += [ReLU()] models += [nn.Conv2DTranspose(4, 4, 3, groups=4, padding=1)] models += [nn.BatchNorm2D(4)] models += [ReLU()] models += [nn.Conv2D(4, 3, 3)] models += [ReLU()] models = ofa_super.convert(models) models += [ Block( SuperSeparableConv2D( 3, 6, 1, padding=1, candidate_config={'channel': (3, 6)}), fixed=True) ] with supernet( kernel_size=(3, 5, 7), expand_ratio=(1, 2, 4)) as ofa_super: models1 = [] models1 += [nn.Conv2D(6, 4, 3)] models1 += [nn.BatchNorm2D(4)] models1 += [ReLU()] models1 += [nn.Conv2D(4, 4, 3, groups=2)] models1 += [nn.InstanceNorm2D(4)] models1 += [ReLU()] models1 += [nn.Conv2DTranspose(4, 4, 3, groups=2)] models1 += [nn.BatchNorm2D(4)] models1 += [ReLU()] models1 += [nn.Conv2DTranspose(4, 4, 3)] models1 += [nn.BatchNorm2D(4)] models1 += [ReLU()] models1 += [nn.Conv2DTranspose(4, 4, 1)] models1 += [nn.BatchNorm2D(4)] models1 += [ReLU()] models1 = ofa_super.convert(models1) models += models1 self.models = paddle.nn.Sequential(*models)
def __init__(self, in_channels, out_channels, kernel_size, stride, padding, num_groups=1, act='relu', conv_lr=1., conv_decay=0., norm_decay=0., norm_type='bn', name=None): super(ConvBNLayer, self).__init__() self.act = act self._conv = nn.Conv2D( in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, groups=num_groups, weight_attr=ParamAttr( learning_rate=conv_lr, initializer=KaimingNormal(), regularizer=L2Decay(conv_decay), name=name + "_weights"), bias_attr=False) param_attr = ParamAttr( name=name + "_bn_scale", regularizer=L2Decay(norm_decay)) bias_attr = ParamAttr( name=name + "_bn_offset", regularizer=L2Decay(norm_decay)) if norm_type == 'sync_bn': self._batch_norm = nn.SyncBatchNorm( out_channels, weight_attr=param_attr, bias_attr=bias_attr) else: self._batch_norm = nn.BatchNorm( out_channels, act=None, param_attr=param_attr, bias_attr=bias_attr, use_global_stats=False, moving_mean_name=name + '_bn_mean', moving_variance_name=name + '_bn_variance')
def __init__(self, pretrained, input_channels=3, num_classes=19, multi_grid=False): super(Glore, self).__init__() self.resnet = ResNet101(pretrained=pretrained, multi_grid=True) self.num_classes = num_classes self.gru_module = GruModule(input_channel=512, num_state=128, num_node=64) self.auxhead = AuxHead(1024, self.num_classes) self.dropout = fluid.dygraph.Dropout(0.1) learning_rate = 10.0 if cfg.DATASET.DATASET_NAME == 'cityscapes': self.connect_conv = fluid.dygraph.Conv2D( 2048, 512, 1, bias_attr=False, param_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) else: self.connect_conv = fluid.dygraph.Conv2D( 2048, 512, 3, bias_attr=False, param_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self.connect_bn = nn.SyncBatchNorm(512) self.connect_relu = nn.ReLU() param_attr = fluid.ParamAttr( regularizer=fluid.regularizer.L2DecayRegularizer( regularization_coeff=0.0), initializer=fluid.initializer.TruncatedNormal(loc=0.0, scale=0.01), learning_rate=10.0) self.get_logit_conv = fluid.dygraph.Conv2D(512, num_classes, filter_size=1, param_attr=param_attr, bias_attr=True)
def __init__(self, fea_dim, num_classes, name_scope=''): super(PSPHead, self).__init__(name_scope) self.num_classes = num_classes self.sizes = (1, 2, 3, 6) self.layers = [] learning_rate = 10.0 for size in self.sizes: path = MakePath('psp_' + str(size), fea_dim, fea_dim / len(self.sizes)) self.layers.append( self.add_sublayer('_{}'.format('psp_' + str(size) + '_path'), path)) fea_dim = fea_dim * 2 self.pre_cls_conv = fluid.dygraph.Conv2D( fea_dim, 512, filter_size=3, stride=1, padding=1, bias_attr=False, param_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self.pre_cls_bn = nn.SyncBatchNorm( 512, weight_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), bias_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self.pre_drop_out = fluid.dygraph.Dropout(p=0.1) self.cls_conv = fluid.dygraph.Conv2D( 512, num_classes, filter_size=1, param_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), bias_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), )
def __init__(self, input_channel=512, num_state=128, num_node=64, normalize=False): super(GruModule, self).__init__() self.Normalize = normalize self.num_state = 128 # node channels self.num_node = 64 # node number self.reduction_dim = fluid.dygraph.Conv2D(input_channel, num_state, filter_size=1) self.projection_mat = fluid.dygraph.Conv2D(input_channel, num_node, filter_size=1) self.gcn = GCN(num_state=self.num_state, num_node=self.num_node) self.extend_dim = fluid.dygraph.Conv2D(self.num_state, input_channel, filter_size=1, bias_attr=False) self.extend_bn = nn.SyncBatchNorm(input_channel, epsilon=1e-4)
def __init__(self, in_channels, out_channels=None, kernel_size=3, norm_type='bn', norm_groups=32, act='swish'): super(SeparableConvLayer, self).__init__() assert norm_type in ['bn', 'sync_bn', 'gn', None] assert act in ['swish', 'relu', None] self.in_channels = in_channels if out_channels is None: self.out_channels = self.in_channels self.norm_type = norm_type self.norm_groups = norm_groups self.depthwise_conv = nn.Conv2D(in_channels, in_channels, kernel_size, padding=kernel_size // 2, groups=in_channels, bias_attr=False) self.pointwise_conv = nn.Conv2D(in_channels, self.out_channels, 1) # norm type if self.norm_type == 'bn': self.norm = nn.BatchNorm2D(self.out_channels) elif self.norm_type == 'sync_bn': self.norm = nn.SyncBatchNorm(self.out_channels) elif self.norm_type == 'gn': self.norm = nn.GroupNorm(num_groups=self.norm_groups, num_channels=self.out_channels) # activation if act == 'swish': self.act = nn.Swish() elif act == 'relu': self.act = nn.ReLU()
def __init__(self, name_scope, fea_dim, reduction_dim): super(MakePath, self).__init__(name_scope) #conv2d\bn\Relu learning_rate = 10.0 assert reduction_dim == int( reduction_dim ), 'the input dims of psphead cannot be fully devided by path number, please check!' reduction_dim = int(reduction_dim) self.conv = fluid.dygraph.Conv2D(fea_dim, reduction_dim, 1, 1, bias_attr=False, param_attr=fluid.ParamAttr( learning_rate=learning_rate, trainable=True)) self.bn = nn.SyncBatchNorm( reduction_dim, bias_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), weight_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self.act = nn.ReLU()
def __init__(self, name_scope, input_channels, num_filters, filter_size=3, stride=1, dilation=1, act=None, learning_rate=1.0, dtype='float32', bias_attr=False): super(ConvBN, self).__init__(name_scope) if dilation != 1: padding = dilation else: padding = (filter_size - 1) // 2 self._conv = nn.Conv2D( in_channels=input_channels, out_channels=num_filters, kernel_size=filter_size, stride=stride, padding=padding, dilation=dilation, bias_attr=bias_attr if bias_attr is False else fluid.ParamAttr( learning_rate=learning_rate, trainable=True), weight_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self._bn = nn.SyncBatchNorm( num_filters, momentum=0.9, epsilon=1e-5, bias_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True), weight_attr=fluid.ParamAttr(learning_rate=learning_rate, trainable=True)) self.act = act
def SyncBatchNorm(*args, **kwargs): """In cpu environment nn.SyncBatchNorm does not have kernel so use nn.BatchNorm2D instead""" if paddle.get_device() == 'cpu': return nn.BatchNorm2D(*args, **kwargs) else: return nn.SyncBatchNorm(*args, **kwargs)
def SyncBatchNorm(*args, **kwargs): if paddle.distributed.get_world_size() > 1: return nn.SyncBatchNorm(*args, **kwargs) else: return nn.BatchNorm(*args, **kwargs)
def __init__(self, ch_in, ch_out, filter_size, stride, groups=1, norm_type='bn', norm_decay=0., norm_groups=32, use_dcn=False, bias_on=False, lr_scale=1., freeze_norm=False, initializer=Normal(mean=0., std=0.01)): super(ConvNormLayer, self).__init__() assert norm_type in ['bn', 'sync_bn', 'gn'] if bias_on: bias_attr = ParamAttr(initializer=Constant(value=0.), learning_rate=lr_scale) else: bias_attr = False if not use_dcn: self.conv = nn.Conv2D(in_channels=ch_in, out_channels=ch_out, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=groups, weight_attr=ParamAttr( initializer=initializer, learning_rate=1.), bias_attr=bias_attr) else: # in FCOS-DCN head, specifically need learning_rate and regularizer self.conv = DeformableConvV2(in_channels=ch_in, out_channels=ch_out, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=groups, weight_attr=ParamAttr( initializer=initializer, learning_rate=1.), bias_attr=True, lr_scale=2., regularizer=L2Decay(norm_decay)) norm_lr = 0. if freeze_norm else 1. param_attr = ParamAttr(learning_rate=norm_lr, regularizer=L2Decay(norm_decay)) bias_attr = ParamAttr(learning_rate=norm_lr, regularizer=L2Decay(norm_decay)) if norm_type == 'bn': self.norm = nn.BatchNorm2D(ch_out, weight_attr=param_attr, bias_attr=bias_attr) elif norm_type == 'sync_bn': self.norm = nn.SyncBatchNorm(ch_out, weight_attr=param_attr, bias_attr=bias_attr) elif norm_type == 'gn': self.norm = nn.GroupNorm(num_groups=norm_groups, num_channels=ch_out, weight_attr=param_attr, bias_attr=bias_attr)
def __init__(self, ch_in, ch_out, filter_size, stride, groups=1, act=None, norm_type='bn', norm_decay=0., freeze_norm=True, lr=1.0, dcn_v2=False): super(ConvNormLayer, self).__init__() assert norm_type in ['bn', 'sync_bn'] self.norm_type = norm_type self.act = act self.dcn_v2 = dcn_v2 if not self.dcn_v2: self.conv = nn.Conv2D(in_channels=ch_in, out_channels=ch_out, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=groups, weight_attr=ParamAttr(learning_rate=lr), bias_attr=False) else: self.offset_channel = 2 * filter_size**2 self.mask_channel = filter_size**2 self.conv_offset = nn.Conv2D( in_channels=ch_in, out_channels=3 * filter_size**2, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, weight_attr=ParamAttr(initializer=Constant(0.)), bias_attr=ParamAttr(initializer=Constant(0.))) self.conv = DeformConv2D(in_channels=ch_in, out_channels=ch_out, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, dilation=1, groups=groups, weight_attr=ParamAttr(learning_rate=lr), bias_attr=False) norm_lr = 0. if freeze_norm else lr param_attr = ParamAttr(learning_rate=norm_lr, regularizer=L2Decay(norm_decay), trainable=False if freeze_norm else True) bias_attr = ParamAttr(learning_rate=norm_lr, regularizer=L2Decay(norm_decay), trainable=False if freeze_norm else True) global_stats = True if freeze_norm else False if norm_type == 'sync_bn': self.norm = nn.SyncBatchNorm(ch_out, weight_attr=param_attr, bias_attr=bias_attr) else: self.norm = nn.BatchNorm(ch_out, act=None, param_attr=param_attr, bias_attr=bias_attr, use_global_stats=global_stats) norm_params = self.norm.parameters() if freeze_norm: for param in norm_params: param.stop_gradient = True
def func_test_layer_str(self): module = nn.ELU(0.2) self.assertEqual(str(module), 'ELU(alpha=0.2)') module = nn.CELU(0.2) self.assertEqual(str(module), 'CELU(alpha=0.2)') module = nn.GELU(True) self.assertEqual(str(module), 'GELU(approximate=True)') module = nn.Hardshrink() self.assertEqual(str(module), 'Hardshrink(threshold=0.5)') module = nn.Hardswish(name="Hardswish") self.assertEqual(str(module), 'Hardswish(name=Hardswish)') module = nn.Tanh(name="Tanh") self.assertEqual(str(module), 'Tanh(name=Tanh)') module = nn.Hardtanh(name="Hardtanh") self.assertEqual(str(module), 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)') module = nn.PReLU(1, 0.25, name="PReLU", data_format="NCHW") self.assertEqual( str(module), 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)' ) module = nn.ReLU() self.assertEqual(str(module), 'ReLU()') module = nn.ReLU6() self.assertEqual(str(module), 'ReLU6()') module = nn.SELU() self.assertEqual( str(module), 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)') module = nn.LeakyReLU() self.assertEqual(str(module), 'LeakyReLU(negative_slope=0.01)') module = nn.Sigmoid() self.assertEqual(str(module), 'Sigmoid()') module = nn.Hardsigmoid() self.assertEqual(str(module), 'Hardsigmoid()') module = nn.Softplus() self.assertEqual(str(module), 'Softplus(beta=1, threshold=20)') module = nn.Softshrink() self.assertEqual(str(module), 'Softshrink(threshold=0.5)') module = nn.Softsign() self.assertEqual(str(module), 'Softsign()') module = nn.Swish() self.assertEqual(str(module), 'Swish()') module = nn.Tanhshrink() self.assertEqual(str(module), 'Tanhshrink()') module = nn.ThresholdedReLU() self.assertEqual(str(module), 'ThresholdedReLU(threshold=1.0)') module = nn.LogSigmoid() self.assertEqual(str(module), 'LogSigmoid()') module = nn.Softmax() self.assertEqual(str(module), 'Softmax(axis=-1)') module = nn.LogSoftmax() self.assertEqual(str(module), 'LogSoftmax(axis=-1)') module = nn.Maxout(groups=2) self.assertEqual(str(module), 'Maxout(groups=2, axis=1)') module = nn.Linear(2, 4, name='linear') self.assertEqual( str(module), 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)' ) module = nn.Upsample(size=[12, 12]) self.assertEqual( str(module), 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)' ) module = nn.UpsamplingNearest2D(size=[12, 12]) self.assertEqual( str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)') module = nn.UpsamplingBilinear2D(size=[12, 12]) self.assertEqual( str(module), 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)') module = nn.Bilinear(in1_features=5, in2_features=4, out_features=1000) self.assertEqual( str(module), 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)' ) module = nn.Dropout(p=0.5) self.assertEqual(str(module), 'Dropout(p=0.5, axis=None, mode=upscale_in_train)') module = nn.Dropout2D(p=0.5) self.assertEqual(str(module), 'Dropout2D(p=0.5, data_format=NCHW)') module = nn.Dropout3D(p=0.5) self.assertEqual(str(module), 'Dropout3D(p=0.5, data_format=NCDHW)') module = nn.AlphaDropout(p=0.5) self.assertEqual(str(module), 'AlphaDropout(p=0.5)') module = nn.Pad1D(padding=[1, 2], mode='constant') self.assertEqual( str(module), 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)') module = nn.Pad2D(padding=[1, 0, 1, 2], mode='constant') self.assertEqual( str(module), 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)' ) module = nn.ZeroPad2D(padding=[1, 0, 1, 2]) self.assertEqual(str(module), 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)') module = nn.Pad3D(padding=[1, 0, 1, 2, 0, 0], mode='constant') self.assertEqual( str(module), 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)' ) module = nn.CosineSimilarity(axis=0) self.assertEqual(str(module), 'CosineSimilarity(axis=0, eps=1e-08)') module = nn.Embedding(10, 3, sparse=True) self.assertEqual(str(module), 'Embedding(10, 3, sparse=True)') module = nn.Conv1D(3, 2, 3) self.assertEqual(str(module), 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)') module = nn.Conv1DTranspose(2, 1, 2) self.assertEqual( str(module), 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)') module = nn.Conv2D(4, 6, (3, 3)) self.assertEqual(str(module), 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)') module = nn.Conv2DTranspose(4, 6, (3, 3)) self.assertEqual( str(module), 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)') module = nn.Conv3D(4, 6, (3, 3, 3)) self.assertEqual( str(module), 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') module = nn.Conv3DTranspose(4, 6, (3, 3, 3)) self.assertEqual( str(module), 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') module = nn.PairwiseDistance() self.assertEqual(str(module), 'PairwiseDistance(p=2.0)') module = nn.InstanceNorm1D(2) self.assertEqual(str(module), 'InstanceNorm1D(num_features=2, epsilon=1e-05)') module = nn.InstanceNorm2D(2) self.assertEqual(str(module), 'InstanceNorm2D(num_features=2, epsilon=1e-05)') module = nn.InstanceNorm3D(2) self.assertEqual(str(module), 'InstanceNorm3D(num_features=2, epsilon=1e-05)') module = nn.GroupNorm(num_channels=6, num_groups=6) self.assertEqual( str(module), 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)') module = nn.LayerNorm([2, 2, 3]) self.assertEqual( str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)') module = nn.BatchNorm1D(1) self.assertEqual( str(module), 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)' ) module = nn.BatchNorm2D(1) self.assertEqual( str(module), 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)') module = nn.BatchNorm3D(1) self.assertEqual( str(module), 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)' ) module = nn.SyncBatchNorm(2) self.assertEqual( str(module), 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)') module = nn.LocalResponseNorm(size=5) self.assertEqual( str(module), 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)') module = nn.AvgPool1D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool1D(kernel_size=2, stride=2, padding=0)') module = nn.AvgPool2D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool2D(kernel_size=2, stride=2, padding=0)') module = nn.AvgPool3D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool3D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool1D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool1D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool2D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool2D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool3D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool3D(kernel_size=2, stride=2, padding=0)') module = nn.AdaptiveAvgPool1D(output_size=16) self.assertEqual(str(module), 'AdaptiveAvgPool1D(output_size=16)') module = nn.AdaptiveAvgPool2D(output_size=3) self.assertEqual(str(module), 'AdaptiveAvgPool2D(output_size=3)') module = nn.AdaptiveAvgPool3D(output_size=3) self.assertEqual(str(module), 'AdaptiveAvgPool3D(output_size=3)') module = nn.AdaptiveMaxPool1D(output_size=16, return_mask=True) self.assertEqual( str(module), 'AdaptiveMaxPool1D(output_size=16, return_mask=True)') module = nn.AdaptiveMaxPool2D(output_size=3, return_mask=True) self.assertEqual(str(module), 'AdaptiveMaxPool2D(output_size=3, return_mask=True)') module = nn.AdaptiveMaxPool3D(output_size=3, return_mask=True) self.assertEqual(str(module), 'AdaptiveMaxPool3D(output_size=3, return_mask=True)') module = nn.SimpleRNNCell(16, 32) self.assertEqual(str(module), 'SimpleRNNCell(16, 32)') module = nn.LSTMCell(16, 32) self.assertEqual(str(module), 'LSTMCell(16, 32)') module = nn.GRUCell(16, 32) self.assertEqual(str(module), 'GRUCell(16, 32)') module = nn.PixelShuffle(3) self.assertEqual(str(module), 'PixelShuffle(upscale_factor=3)') module = nn.SimpleRNN(16, 32, 2) self.assertEqual( str(module), 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)' ) module = nn.LSTM(16, 32, 2) self.assertEqual( str(module), 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)' ) module = nn.GRU(16, 32, 2) self.assertEqual( str(module), 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)' ) module1 = nn.Sequential( ('conv1', nn.Conv2D(1, 20, 5)), ('relu1', nn.ReLU()), ('conv2', nn.Conv2D(20, 64, 5)), ('relu2', nn.ReLU())) self.assertEqual( str(module1), 'Sequential(\n '\ '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n '\ '(relu1): ReLU()\n '\ '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n '\ '(relu2): ReLU()\n)' ) module2 = nn.Sequential( nn.Conv3DTranspose(4, 6, (3, 3, 3)), nn.AvgPool3D(kernel_size=2, stride=2, padding=0), nn.Tanh(name="Tanh"), module1, nn.Conv3D(4, 6, (3, 3, 3)), nn.MaxPool3D(kernel_size=2, stride=2, padding=0), nn.GELU(True)) self.assertEqual( str(module2), 'Sequential(\n '\ '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n '\ '(2): Tanh(name=Tanh)\n '\ '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n'\ ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n '\ '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n '\ '(6): GELU(approximate=True)\n)' )
def __init__(self, ch_in, ch_out, filter_size, stride, name_adapter, groups=1, act=None, norm_type='bn', norm_decay=0., freeze_norm=True, lr=1.0, dcn_v2=False, name=None): super(ConvNormLayer, self).__init__() assert norm_type in ['bn', 'sync_bn'] self.norm_type = norm_type self.act = act if not dcn_v2: self.conv = nn.Conv2D( in_channels=ch_in, out_channels=ch_out, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=groups, weight_attr=paddle.ParamAttr( learning_rate=lr, name=name + "_weights"), bias_attr=False) else: self.conv = DeformableConvV2( in_channels=ch_in, out_channels=ch_out, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=groups, weight_attr=paddle.ParamAttr( learning_rate=lr, name=name + '_weights'), bias_attr=False, name=name) bn_name = name_adapter.fix_conv_norm_name(name) norm_lr = 0. if freeze_norm else lr param_attr = paddle.ParamAttr( learning_rate=norm_lr, regularizer=L2Decay(norm_decay), name=bn_name + "_scale", trainable=False if freeze_norm else True) bias_attr = paddle.ParamAttr( learning_rate=norm_lr, regularizer=L2Decay(norm_decay), name=bn_name + "_offset", trainable=False if freeze_norm else True) global_stats = True if freeze_norm else False if norm_type == 'sync_bn': self.norm = nn.SyncBatchNorm( ch_out, weight_attr=param_attr, bias_attr=bias_attr) else: self.norm = nn.BatchNorm( ch_out, act=None, param_attr=param_attr, bias_attr=bias_attr, use_global_stats=global_stats, moving_mean_name=bn_name + '_mean', moving_variance_name=bn_name + '_variance') norm_params = self.norm.parameters() if freeze_norm: for param in norm_params: param.stop_gradient = True