def __init__(self, config, last_channel, scale=1.0, num_classes=1000, with_pool=True): super().__init__() self.config = config self.scale = scale self.last_channel = last_channel self.num_classes = num_classes self.with_pool = with_pool self.firstconv_in_channels = config[0].in_channels self.lastconv_in_channels = config[-1].in_channels self.lastconv_out_channels = self.lastconv_in_channels * 6 norm_layer = partial(nn.BatchNorm2D, epsilon=0.001, momentum=0.99) self.conv = ConvNormActivation(in_channels=3, out_channels=self.firstconv_in_channels, kernel_size=3, stride=2, padding=1, groups=1, activation_layer=nn.Hardswish, norm_layer=norm_layer) self.blocks = nn.Sequential(*[ InvertedResidual(in_channels=cfg.in_channels, expanded_channels=cfg.expanded_channels, out_channels=cfg.out_channels, filter_size=cfg.kernel, stride=cfg.stride, use_se=cfg.use_se, activation_layer=cfg.activation_layer, norm_layer=norm_layer) for cfg in self.config ]) self.lastconv = ConvNormActivation( in_channels=self.lastconv_in_channels, out_channels=self.lastconv_out_channels, kernel_size=1, stride=1, padding=0, groups=1, norm_layer=norm_layer, activation_layer=nn.Hardswish) if with_pool: self.avgpool = nn.AdaptiveAvgPool2D(1) if num_classes > 0: self.classifier = nn.Sequential( nn.Linear(self.lastconv_out_channels, self.last_channel), nn.Hardswish(), nn.Dropout(p=0.2), nn.Linear(self.last_channel, num_classes))
def _create_act(act): if act == "hardswish": return nn.Hardswish() elif act == "relu": return nn.ReLU() elif act is None: return Identity() else: raise RuntimeError( "The activation function is not supported: {}".format(act))
def __init__(self, class_num=1000, scale=1.0, dropout_prob=0.2, class_expand=1280): super().__init__() self.scale = scale self.class_num = class_num self.class_expand = class_expand stage_repeats = [3, 7, 3] stage_out_channels = [ -1, 24, make_divisible(116 * scale), make_divisible(232 * scale), make_divisible(464 * scale), 1024 ] self.conv1 = ConvBNLayer(in_channels=3, out_channels=stage_out_channels[1], kernel_size=3, stride=2) self.max_pool = MaxPool2D(kernel_size=3, stride=2, padding=1) block_list = [] for stage_id, num_repeat in enumerate(stage_repeats): for i in range(num_repeat): if i == 0: block = ESBlock2( in_channels=stage_out_channels[stage_id + 1], out_channels=stage_out_channels[stage_id + 2]) else: block = ESBlock1( in_channels=stage_out_channels[stage_id + 2], out_channels=stage_out_channels[stage_id + 2]) block_list.append(block) self.blocks = nn.Sequential(*block_list) self.conv2 = ConvBNLayer(in_channels=stage_out_channels[-2], out_channels=stage_out_channels[-1], kernel_size=1) self.avg_pool = AdaptiveAvgPool2D(1) self.last_conv = Conv2D(in_channels=stage_out_channels[-1], out_channels=self.class_expand, kernel_size=1, stride=1, padding=0, bias_attr=False) self.hardswish = nn.Hardswish() self.dropout = Dropout(p=dropout_prob, mode="downscale_in_infer") self.flatten = nn.Flatten(start_axis=1, stop_axis=-1) self.fc = Linear(self.class_expand, self.class_num)
def __init__(self, num_channels, filter_size, num_filters, stride, num_groups=1): super().__init__() self.conv = Conv2D(in_channels=num_channels, out_channels=num_filters, kernel_size=filter_size, stride=stride, padding=(filter_size - 1) // 2, groups=num_groups, weight_attr=ParamAttr(initializer=KaimingNormal()), bias_attr=False) self.bn = BatchNorm(num_filters, param_attr=ParamAttr(regularizer=L2Decay(0.0)), bias_attr=ParamAttr(regularizer=L2Decay(0.0))) self.hardswish = nn.Hardswish()
def __init__(self, in_channels, out_channels, kernel_size, stride=1, groups=1, if_act=True): super().__init__() self.conv = Conv2D(in_channels=in_channels, out_channels=out_channels, kernel_size=kernel_size, stride=stride, padding=(kernel_size - 1) // 2, groups=groups, weight_attr=ParamAttr(initializer=KaimingNormal()), bias_attr=False) self.bn = BatchNorm(out_channels, param_attr=ParamAttr(regularizer=L2Decay(0.0)), bias_attr=ParamAttr(regularizer=L2Decay(0.0))) self.if_act = if_act self.hardswish = nn.Hardswish()
def get_activation_layer(activation): """ Create activation layer from string/function. Parameters: ---------- activation : function, or str, or nn.Module Activation function or name of activation function. Returns: ------- nn.Module Activation layer. """ assert activation is not None if isfunction(activation): return activation() elif isinstance(activation, str): if activation == "relu": return nn.ReLU() elif activation == "relu6": return nn.ReLU6() elif activation == "swish": return nn.Swish() elif activation == "hswish": return nn.Hardswish() elif activation == "sigmoid": return nn.Sigmoid() elif activation == "hsigmoid": return nn.Hardsigmoid() elif activation == "identity": return Identity() else: raise NotImplementedError() else: assert isinstance(activation, nn.Layer) return activation
def __init__( self, inverted_residual_setting: List[InvertedResidualConfig], last_channel: int, num_classes: int = 1000, block: Optional[Callable[..., nn.Layer]] = None, norm_layer: Optional[Callable[..., nn.Layer]] = None, dropout: float = 0.2, **kwargs: Any, ) -> None: """ MobileNet V3 main class Args: inverted_residual_setting (List[InvertedResidualConfig]): Network structure last_channel (int): The number of channels on the penultimate layer num_classes (int): Number of classes block (Optional[Callable[..., nn.Layer]]): Module specifying inverted residual building block for mobilenet norm_layer (Optional[Callable[..., nn.Layer]]): Module specifying the normalization layer to use dropout (float): The droupout probability """ super().__init__() if not inverted_residual_setting: raise ValueError( "The inverted_residual_setting should not be empty") elif not (isinstance(inverted_residual_setting, Sequence) and all([ isinstance(s, InvertedResidualConfig) for s in inverted_residual_setting ])): raise TypeError( "The inverted_residual_setting should be List[InvertedResidualConfig]" ) if block is None: block = InvertedResidual if norm_layer is None: norm_layer = partial(nn.BatchNorm2D, epsilon=0.001, momentum=0.01) layers: List[nn.Layer] = [] # building first layer firstconv_output_channels = inverted_residual_setting[0].input_channels layers.append( ConvNormActivation( 3, firstconv_output_channels, kernel_size=3, stride=2, norm_layer=norm_layer, activation_layer=nn.Hardswish, )) # building inverted residual blocks for cnf in inverted_residual_setting: layers.append(block(cnf, norm_layer)) # building last several layers lastconv_input_channels = inverted_residual_setting[-1].out_channels lastconv_output_channels = 6 * lastconv_input_channels layers.append( ConvNormActivation( lastconv_input_channels, lastconv_output_channels, kernel_size=1, norm_layer=norm_layer, activation_layer=nn.Hardswish, )) self.features = nn.Sequential(*layers) self.avgpool = nn.AdaptiveAvgPool2D(1) self.classifier = nn.Sequential( nn.Linear(lastconv_output_channels, last_channel), nn.Hardswish(), nn.Dropout(p=dropout), nn.Linear(last_channel, num_classes), )
def __init__(self, config, stages_pattern, scale=1.0, class_num=1000, inplanes=STEM_CONV_NUMBER, class_squeeze=LAST_SECOND_CONV_LARGE, class_expand=LAST_CONV, dropout_prob=0.2, return_patterns=None, return_stages=None): super().__init__() self.cfg = config self.scale = scale self.inplanes = inplanes self.class_squeeze = class_squeeze self.class_expand = class_expand self.class_num = class_num self.conv = ConvBNLayer( in_c=3, out_c=_make_divisible(self.inplanes * self.scale), filter_size=3, stride=2, padding=1, num_groups=1, if_act=True, act="hardswish") self.blocks = nn.Sequential(* [ ResidualUnit( in_c=_make_divisible(self.inplanes * self.scale if i == 0 else self.cfg[i - 1][2] * self.scale), mid_c=_make_divisible(self.scale * exp), out_c=_make_divisible(self.scale * c), filter_size=k, stride=s, use_se=se, act=act) for i, (k, exp, c, se, act, s) in enumerate(self.cfg) ]) self.last_second_conv = ConvBNLayer( in_c=_make_divisible(self.cfg[-1][2] * self.scale), out_c=_make_divisible(self.scale * self.class_squeeze), filter_size=1, stride=1, padding=0, num_groups=1, if_act=True, act="hardswish") self.avg_pool = AdaptiveAvgPool2D(1) self.last_conv = Conv2D( in_channels=_make_divisible(self.scale * self.class_squeeze), out_channels=self.class_expand, kernel_size=1, stride=1, padding=0, bias_attr=False) self.hardswish = nn.Hardswish() if dropout_prob is not None: self.dropout = Dropout(p=dropout_prob, mode="downscale_in_infer") else: self.dropout = None self.flatten = nn.Flatten(start_axis=1, stop_axis=-1) self.fc = Linear(self.class_expand, class_num) super().init_res( stages_pattern, return_patterns=return_patterns, return_stages=return_stages)
def func_test_layer_str(self): module = nn.ELU(0.2) self.assertEqual(str(module), 'ELU(alpha=0.2)') module = nn.CELU(0.2) self.assertEqual(str(module), 'CELU(alpha=0.2)') module = nn.GELU(True) self.assertEqual(str(module), 'GELU(approximate=True)') module = nn.Hardshrink() self.assertEqual(str(module), 'Hardshrink(threshold=0.5)') module = nn.Hardswish(name="Hardswish") self.assertEqual(str(module), 'Hardswish(name=Hardswish)') module = nn.Tanh(name="Tanh") self.assertEqual(str(module), 'Tanh(name=Tanh)') module = nn.Hardtanh(name="Hardtanh") self.assertEqual(str(module), 'Hardtanh(min=-1.0, max=1.0, name=Hardtanh)') module = nn.PReLU(1, 0.25, name="PReLU", data_format="NCHW") self.assertEqual( str(module), 'PReLU(num_parameters=1, data_format=NCHW, init=0.25, dtype=float32, name=PReLU)' ) module = nn.ReLU() self.assertEqual(str(module), 'ReLU()') module = nn.ReLU6() self.assertEqual(str(module), 'ReLU6()') module = nn.SELU() self.assertEqual( str(module), 'SELU(scale=1.0507009873554805, alpha=1.6732632423543772)') module = nn.LeakyReLU() self.assertEqual(str(module), 'LeakyReLU(negative_slope=0.01)') module = nn.Sigmoid() self.assertEqual(str(module), 'Sigmoid()') module = nn.Hardsigmoid() self.assertEqual(str(module), 'Hardsigmoid()') module = nn.Softplus() self.assertEqual(str(module), 'Softplus(beta=1, threshold=20)') module = nn.Softshrink() self.assertEqual(str(module), 'Softshrink(threshold=0.5)') module = nn.Softsign() self.assertEqual(str(module), 'Softsign()') module = nn.Swish() self.assertEqual(str(module), 'Swish()') module = nn.Tanhshrink() self.assertEqual(str(module), 'Tanhshrink()') module = nn.ThresholdedReLU() self.assertEqual(str(module), 'ThresholdedReLU(threshold=1.0)') module = nn.LogSigmoid() self.assertEqual(str(module), 'LogSigmoid()') module = nn.Softmax() self.assertEqual(str(module), 'Softmax(axis=-1)') module = nn.LogSoftmax() self.assertEqual(str(module), 'LogSoftmax(axis=-1)') module = nn.Maxout(groups=2) self.assertEqual(str(module), 'Maxout(groups=2, axis=1)') module = nn.Linear(2, 4, name='linear') self.assertEqual( str(module), 'Linear(in_features=2, out_features=4, dtype=float32, name=linear)' ) module = nn.Upsample(size=[12, 12]) self.assertEqual( str(module), 'Upsample(size=[12, 12], mode=nearest, align_corners=False, align_mode=0, data_format=NCHW)' ) module = nn.UpsamplingNearest2D(size=[12, 12]) self.assertEqual( str(module), 'UpsamplingNearest2D(size=[12, 12], data_format=NCHW)') module = nn.UpsamplingBilinear2D(size=[12, 12]) self.assertEqual( str(module), 'UpsamplingBilinear2D(size=[12, 12], data_format=NCHW)') module = nn.Bilinear(in1_features=5, in2_features=4, out_features=1000) self.assertEqual( str(module), 'Bilinear(in1_features=5, in2_features=4, out_features=1000, dtype=float32)' ) module = nn.Dropout(p=0.5) self.assertEqual(str(module), 'Dropout(p=0.5, axis=None, mode=upscale_in_train)') module = nn.Dropout2D(p=0.5) self.assertEqual(str(module), 'Dropout2D(p=0.5, data_format=NCHW)') module = nn.Dropout3D(p=0.5) self.assertEqual(str(module), 'Dropout3D(p=0.5, data_format=NCDHW)') module = nn.AlphaDropout(p=0.5) self.assertEqual(str(module), 'AlphaDropout(p=0.5)') module = nn.Pad1D(padding=[1, 2], mode='constant') self.assertEqual( str(module), 'Pad1D(padding=[1, 2], mode=constant, value=0.0, data_format=NCL)') module = nn.Pad2D(padding=[1, 0, 1, 2], mode='constant') self.assertEqual( str(module), 'Pad2D(padding=[1, 0, 1, 2], mode=constant, value=0.0, data_format=NCHW)' ) module = nn.ZeroPad2D(padding=[1, 0, 1, 2]) self.assertEqual(str(module), 'ZeroPad2D(padding=[1, 0, 1, 2], data_format=NCHW)') module = nn.Pad3D(padding=[1, 0, 1, 2, 0, 0], mode='constant') self.assertEqual( str(module), 'Pad3D(padding=[1, 0, 1, 2, 0, 0], mode=constant, value=0.0, data_format=NCDHW)' ) module = nn.CosineSimilarity(axis=0) self.assertEqual(str(module), 'CosineSimilarity(axis=0, eps=1e-08)') module = nn.Embedding(10, 3, sparse=True) self.assertEqual(str(module), 'Embedding(10, 3, sparse=True)') module = nn.Conv1D(3, 2, 3) self.assertEqual(str(module), 'Conv1D(3, 2, kernel_size=[3], data_format=NCL)') module = nn.Conv1DTranspose(2, 1, 2) self.assertEqual( str(module), 'Conv1DTranspose(2, 1, kernel_size=[2], data_format=NCL)') module = nn.Conv2D(4, 6, (3, 3)) self.assertEqual(str(module), 'Conv2D(4, 6, kernel_size=[3, 3], data_format=NCHW)') module = nn.Conv2DTranspose(4, 6, (3, 3)) self.assertEqual( str(module), 'Conv2DTranspose(4, 6, kernel_size=[3, 3], data_format=NCHW)') module = nn.Conv3D(4, 6, (3, 3, 3)) self.assertEqual( str(module), 'Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') module = nn.Conv3DTranspose(4, 6, (3, 3, 3)) self.assertEqual( str(module), 'Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)') module = nn.PairwiseDistance() self.assertEqual(str(module), 'PairwiseDistance(p=2.0)') module = nn.InstanceNorm1D(2) self.assertEqual(str(module), 'InstanceNorm1D(num_features=2, epsilon=1e-05)') module = nn.InstanceNorm2D(2) self.assertEqual(str(module), 'InstanceNorm2D(num_features=2, epsilon=1e-05)') module = nn.InstanceNorm3D(2) self.assertEqual(str(module), 'InstanceNorm3D(num_features=2, epsilon=1e-05)') module = nn.GroupNorm(num_channels=6, num_groups=6) self.assertEqual( str(module), 'GroupNorm(num_groups=6, num_channels=6, epsilon=1e-05)') module = nn.LayerNorm([2, 2, 3]) self.assertEqual( str(module), 'LayerNorm(normalized_shape=[2, 2, 3], epsilon=1e-05)') module = nn.BatchNorm1D(1) self.assertEqual( str(module), 'BatchNorm1D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCL)' ) module = nn.BatchNorm2D(1) self.assertEqual( str(module), 'BatchNorm2D(num_features=1, momentum=0.9, epsilon=1e-05)') module = nn.BatchNorm3D(1) self.assertEqual( str(module), 'BatchNorm3D(num_features=1, momentum=0.9, epsilon=1e-05, data_format=NCDHW)' ) module = nn.SyncBatchNorm(2) self.assertEqual( str(module), 'SyncBatchNorm(num_features=2, momentum=0.9, epsilon=1e-05)') module = nn.LocalResponseNorm(size=5) self.assertEqual( str(module), 'LocalResponseNorm(size=5, alpha=0.0001, beta=0.75, k=1.0)') module = nn.AvgPool1D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool1D(kernel_size=2, stride=2, padding=0)') module = nn.AvgPool2D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool2D(kernel_size=2, stride=2, padding=0)') module = nn.AvgPool3D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'AvgPool3D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool1D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool1D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool2D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool2D(kernel_size=2, stride=2, padding=0)') module = nn.MaxPool3D(kernel_size=2, stride=2, padding=0) self.assertEqual(str(module), 'MaxPool3D(kernel_size=2, stride=2, padding=0)') module = nn.AdaptiveAvgPool1D(output_size=16) self.assertEqual(str(module), 'AdaptiveAvgPool1D(output_size=16)') module = nn.AdaptiveAvgPool2D(output_size=3) self.assertEqual(str(module), 'AdaptiveAvgPool2D(output_size=3)') module = nn.AdaptiveAvgPool3D(output_size=3) self.assertEqual(str(module), 'AdaptiveAvgPool3D(output_size=3)') module = nn.AdaptiveMaxPool1D(output_size=16, return_mask=True) self.assertEqual( str(module), 'AdaptiveMaxPool1D(output_size=16, return_mask=True)') module = nn.AdaptiveMaxPool2D(output_size=3, return_mask=True) self.assertEqual(str(module), 'AdaptiveMaxPool2D(output_size=3, return_mask=True)') module = nn.AdaptiveMaxPool3D(output_size=3, return_mask=True) self.assertEqual(str(module), 'AdaptiveMaxPool3D(output_size=3, return_mask=True)') module = nn.SimpleRNNCell(16, 32) self.assertEqual(str(module), 'SimpleRNNCell(16, 32)') module = nn.LSTMCell(16, 32) self.assertEqual(str(module), 'LSTMCell(16, 32)') module = nn.GRUCell(16, 32) self.assertEqual(str(module), 'GRUCell(16, 32)') module = nn.PixelShuffle(3) self.assertEqual(str(module), 'PixelShuffle(upscale_factor=3)') module = nn.SimpleRNN(16, 32, 2) self.assertEqual( str(module), 'SimpleRNN(16, 32, num_layers=2\n (0): RNN(\n (cell): SimpleRNNCell(16, 32)\n )\n (1): RNN(\n (cell): SimpleRNNCell(32, 32)\n )\n)' ) module = nn.LSTM(16, 32, 2) self.assertEqual( str(module), 'LSTM(16, 32, num_layers=2\n (0): RNN(\n (cell): LSTMCell(16, 32)\n )\n (1): RNN(\n (cell): LSTMCell(32, 32)\n )\n)' ) module = nn.GRU(16, 32, 2) self.assertEqual( str(module), 'GRU(16, 32, num_layers=2\n (0): RNN(\n (cell): GRUCell(16, 32)\n )\n (1): RNN(\n (cell): GRUCell(32, 32)\n )\n)' ) module1 = nn.Sequential( ('conv1', nn.Conv2D(1, 20, 5)), ('relu1', nn.ReLU()), ('conv2', nn.Conv2D(20, 64, 5)), ('relu2', nn.ReLU())) self.assertEqual( str(module1), 'Sequential(\n '\ '(conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n '\ '(relu1): ReLU()\n '\ '(conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n '\ '(relu2): ReLU()\n)' ) module2 = nn.Sequential( nn.Conv3DTranspose(4, 6, (3, 3, 3)), nn.AvgPool3D(kernel_size=2, stride=2, padding=0), nn.Tanh(name="Tanh"), module1, nn.Conv3D(4, 6, (3, 3, 3)), nn.MaxPool3D(kernel_size=2, stride=2, padding=0), nn.GELU(True)) self.assertEqual( str(module2), 'Sequential(\n '\ '(0): Conv3DTranspose(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ '(1): AvgPool3D(kernel_size=2, stride=2, padding=0)\n '\ '(2): Tanh(name=Tanh)\n '\ '(3): Sequential(\n (conv1): Conv2D(1, 20, kernel_size=[5, 5], data_format=NCHW)\n (relu1): ReLU()\n'\ ' (conv2): Conv2D(20, 64, kernel_size=[5, 5], data_format=NCHW)\n (relu2): ReLU()\n )\n '\ '(4): Conv3D(4, 6, kernel_size=[3, 3, 3], data_format=NCDHW)\n '\ '(5): MaxPool3D(kernel_size=2, stride=2, padding=0)\n '\ '(6): GELU(approximate=True)\n)' )
def __init__(self, scale=1.0, class_expand=1280): super(PPLCNet, self).__init__() self.scale = scale self.class_expand = class_expand self.conv1 = ConvBNLayer( num_channels=3, filter_size=3, num_filters=make_divisible(16 * scale), stride=2) self.blocks2 = nn.Sequential(*[ DepthwiseSeparable( num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks2"]) ]) self.blocks3 = nn.Sequential(*[ DepthwiseSeparable( num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks3"]) ]) self.blocks4 = nn.Sequential(*[ DepthwiseSeparable( num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks4"]) ]) self.blocks5 = nn.Sequential(*[ DepthwiseSeparable( num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks5"]) ]) self.blocks6 = nn.Sequential(*[ DepthwiseSeparable( num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks6"]) ]) self.avg_pool = AdaptiveAvgPool2D(1) self.last_conv = Conv2D( in_channels=make_divisible(NET_CONFIG["blocks6"][-1][2] * scale), out_channels=self.class_expand, kernel_size=1, stride=1, padding=0, bias_attr=False) self.hardswish = nn.Hardswish() self.flatten = nn.Flatten(start_axis=1, stop_axis=-1)
def __init__(self, block_args, global_params): super().__init__() self._block_args = block_args self._bn_mom = global_params.batch_norm_momentum self._bn_eps = global_params.batch_norm_epsilon self.has_se = (self._block_args.se_ratio is not None) and (0 < self._block_args.se_ratio <= 1) self.id_skip = block_args.id_skip # skip connection and drop connect # Get static or dynamic convolution depending on image size Conv2d = get_same_padding_conv2d(image_size=global_params.image_size) # Expansion phase inp = self._block_args.input_filters # number of input channels oup = self._block_args.input_filters * self._block_args.expand_ratio # number of output channels if self._block_args.expand_ratio != 1: self._expand_conv = Conv2d(in_channels=inp, out_channels=oup, kernel_size=1, bias_attr=False) self._bn0 = nn.BatchNorm2D(num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps) # Depthwise convolution phase k = self._block_args.kernel_size s = self._block_args.stride self._depthwise_conv = Conv2d( in_channels=oup, out_channels=oup, groups=oup, # groups makes it depthwise kernel_size=k, stride=s, bias_attr=False) self._bn1 = nn.BatchNorm2D(num_features=oup, momentum=self._bn_mom, epsilon=self._bn_eps) # Squeeze and Excitation layer, if desired if self.has_se: num_squeezed_channels = max( 1, int(self._block_args.input_filters * self._block_args.se_ratio)) self._se_reduce = Conv2d(in_channels=oup, out_channels=num_squeezed_channels, kernel_size=1) self._se_expand = Conv2d(in_channels=num_squeezed_channels, out_channels=oup, kernel_size=1) # Output phase final_oup = self._block_args.output_filters self._project_conv = Conv2d(in_channels=oup, out_channels=final_oup, kernel_size=1, bias_attr=False) self._bn2 = nn.BatchNorm2D(num_features=final_oup, momentum=self._bn_mom, epsilon=self._bn_eps) self._swish = nn.Hardswish()
def set_swish(self, memory_efficient=True): """Sets swish function as memory efficient (for training) or standard (for export)""" self._swish = nn.Hardswish() if memory_efficient else nn.Swish() for block in self._blocks: block.set_swish(memory_efficient)
def __init__(self, blocks_args=None, global_params=None): super().__init__() assert isinstance(blocks_args, list), 'blocks_args should be a list' assert len(blocks_args) > 0, 'block args must be greater than 0' self._global_params = global_params self._blocks_args = blocks_args # Get static or dynamic convolution depending on image size Conv2d = get_same_padding_conv2d(image_size=global_params.image_size) # Batch norm parameters bn_mom = self._global_params.batch_norm_momentum bn_eps = self._global_params.batch_norm_epsilon # Stem in_channels = 3 # rgb out_channels = round_filters( 32, self._global_params) # number of output channels self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias_attr=False) self._bn0 = nn.BatchNorm2D(num_features=out_channels, momentum=bn_mom, epsilon=bn_eps) # Build blocks self._blocks = nn.LayerList([]) for block_args in self._blocks_args: # Update block input and output filters based on depth multiplier. block_args = block_args._replace( input_filters=round_filters(block_args.input_filters, self._global_params), output_filters=round_filters(block_args.output_filters, self._global_params), num_repeat=round_repeats(block_args.num_repeat, self._global_params)) # The first block needs to take care of stride and filter size increase. self._blocks.append(MBConvBlock(block_args, self._global_params)) if block_args.num_repeat > 1: block_args = block_args._replace( input_filters=block_args.output_filters, stride=1) for _ in range(block_args.num_repeat - 1): self._blocks.append( MBConvBlock(block_args, self._global_params)) # Head in_channels = block_args.output_filters # output of final block out_channels = round_filters(1280, self._global_params) self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias_attr=False) self._bn1 = nn.BatchNorm2D(num_features=out_channels, momentum=bn_mom, epsilon=bn_eps) # Final linear layer self._avg_pooling = nn.AdaptiveAvgPool2D(1) self._dropout = nn.Dropout(self._global_params.dropout_rate) self._fc = nn.Linear(out_channels, self._global_params.num_classes) self._swish = nn.Hardswish()
def __init__(self, inverted_residual_setting, last_channel, num_classes=1000, block=None, norm_layer=None, **kwargs): """ MobileNet V3 main class Args: inverted_residual_setting (List[InvertedResidualConfig]): Network structure last_channel (int): The number of channels on the penultimate layer num_classes (int): Number of classes block (Optional[Callable[..., nn.Layer]]): Module specifying inverted residual building block for mobilenet norm_layer (Optional[Callable[..., nn.Layer]]): Module specifying the normalization layer to use """ super().__init__() if not inverted_residual_setting: raise ValueError( "The inverted_residual_setting should not be empty") elif not (isinstance(inverted_residual_setting, Sequence) and all([ isinstance(s, InvertedResidualConfig) for s in inverted_residual_setting ])): raise TypeError( "The inverted_residual_setting should be List[InvertedResidualConfig]" ) if block is None: block = InvertedResidual if norm_layer is None: norm_layer = partial(nn.BatchNorm2d, eps=0.001, momentum=0.01) layers = [] # building first layer firstconv_output_channels = inverted_residual_setting[0].input_channels layers.append( ConvBNActivation(3, firstconv_output_channels, kernel_size=3, stride=2, norm_layer=norm_layer, activation_layer=nn.Hardswish)) # building inverted residual blocks for cnf in inverted_residual_setting: layers.append(block(cnf, norm_layer)) # building last several layers lastconv_input_channels = inverted_residual_setting[-1].out_channels lastconv_output_channels = 6 * lastconv_input_channels layers.append( ConvBNActivation(lastconv_input_channels, lastconv_output_channels, kernel_size=1, norm_layer=norm_layer, activation_layer=nn.Hardswish)) self.features = nn.Sequential(*layers) self.avgpool = nn.AdaptiveAvgPool2d(1) self.classifier = nn.Sequential( nn.Linear(lastconv_output_channels, last_channel), nn.Hardswish(), nn.Dropout(p=0.2, ), nn.Linear(last_channel, num_classes), ) for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out') if m.bias is not None: nn.init.zeros_(m.bias) elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): nn.init.ones_(m.weight) nn.init.zeros_(m.bias) elif isinstance(m, nn.Linear): nn.init.normal_(m.weight, 0, 0.01) nn.init.zeros_(m.bias)
def __init__(self, stages_pattern, scale=1.0, class_num=1000, dropout_prob=0.2, class_expand=1280, return_patterns=None, return_stages=None): super().__init__() self.scale = scale self.class_expand = class_expand self.conv1 = ConvBNLayer(num_channels=3, filter_size=3, num_filters=make_divisible(16 * scale), stride=2) self.blocks2 = nn.Sequential(*[ DepthwiseSeparable(num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks2"]) ]) self.blocks3 = nn.Sequential(*[ DepthwiseSeparable(num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks3"]) ]) self.blocks4 = nn.Sequential(*[ DepthwiseSeparable(num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks4"]) ]) self.blocks5 = nn.Sequential(*[ DepthwiseSeparable(num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks5"]) ]) self.blocks6 = nn.Sequential(*[ DepthwiseSeparable(num_channels=make_divisible(in_c * scale), num_filters=make_divisible(out_c * scale), dw_size=k, stride=s, use_se=se) for i, (k, in_c, out_c, s, se) in enumerate(NET_CONFIG["blocks6"]) ]) self.avg_pool = AdaptiveAvgPool2D(1) self.last_conv = Conv2D(in_channels=make_divisible( NET_CONFIG["blocks6"][-1][2] * scale), out_channels=self.class_expand, kernel_size=1, stride=1, padding=0, bias_attr=False) self.hardswish = nn.Hardswish() self.dropout = Dropout(p=dropout_prob, mode="downscale_in_infer") self.flatten = nn.Flatten(start_axis=1, stop_axis=-1) self.fc = Linear(self.class_expand, class_num) super().init_res(stages_pattern, return_patterns=return_patterns, return_stages=return_stages)