def __init__(self, use_bias, alg, **kwargs): super(ConvAct, self).__init__(**kwargs) self.conv0 = nn.Conv2D(channels=64, kernel_size=(3, 3), strides=1, use_bias=use_bias) if alg == "relu6": self.act = RELU6() elif alg == "leakyrelu": self.act = nn.LeakyReLU(0.25) elif alg == "gelu": self.act = nn.GELU() elif alg == "gelu_tanh": self.act = nn.GELU(approximation='tanh') else: self.act = nn.Activation(activation = alg)
def __init__(self, act_func: str, **kwargs): super(Activation, self).__init__() with self.name_scope(): if act_func in ('relu', 'sigmoid', 'softrelu', 'softsign', 'tanh'): self.act = nn.Activation(act_func) elif act_func == 'leaky': self.act = nn.LeakyReLU(**kwargs) elif act_func == 'prelu': self.act = nn.PReLU(**kwargs) elif act_func == 'selu': self.act = nn.SELU() elif act_func == 'elu': self.act = nn.ELU(**kwargs) elif act_func == 'gelu': self.act = nn.GELU() elif act_func == 'relu6': self.act = ReLU6() elif act_func == 'hard_sigmoid': self.act = HardSigmoid() elif act_func == 'swish': self.act = nn.Swish() elif act_func == 'hard_swish': self.act = HardSwish() elif act_func == 'mish': self.act = Mish() else: raise NotImplementedError( f"Not implemented activation: {act_func}")
def __init__(self, alg, use_bias, **kwargs): super(ConvBNAct, self).__init__(**kwargs) self.conv0 = nn.Conv2D(channels=64, kernel_size=(3, 3), strides=1, use_bias=use_bias) self.bn = nn.BatchNorm() if alg == "relu6": self.act = RELU6() elif alg == "leakyrelu": self.act = nn.LeakyReLU(0.25) elif alg == "gelu": self.act = nn.GELU() else: self.act = nn.Activation(activation = alg)
def __init__(self, use_bias, alg, **kwargs): super(ConvActAdd, self).__init__(**kwargs) self.conv0 = nn.Conv2D(channels=64, kernel_size=(3, 3), strides=1, use_bias=use_bias, weight_initializer=mx.init.Xavier(magnitude=2.24)) if alg == "relu6": self.act = RELU6() elif alg == "leakyrelu": self.act = nn.LeakyReLU(0.25) elif alg == "gelu": self.act = nn.GELU() else: self.act = nn.Activation(activation = alg) self.conv1 = nn.Conv2D(channels=64, kernel_size=(3, 3), strides=1, use_bias=use_bias) self.conv1.share_parameters(self.conv0.collect_params())
def __init__(self, act_type, r, skernel, dilation, channels, useReLU, useGlobal, asBackbone, stride, downsample=False, in_channels=0, norm_layer=BatchNorm, norm_kwargs=None, **kwargs): super(ResBlockV2ATAC, self).__init__(**kwargs) self.bn1 = norm_layer(**({} if norm_kwargs is None else norm_kwargs)) self.conv1 = _conv3x3(channels, stride, in_channels) self.bn2 = norm_layer(**({} if norm_kwargs is None else norm_kwargs)) self.conv2 = _conv3x3(channels, 1, channels) if downsample: self.downsample = nn.Conv2D(channels, 1, stride, use_bias=False, in_channels=in_channels) else: self.downsample = None if act_type == 'relu': self.msAA1 = nn.Activation('relu') self.msAA2 = nn.Activation('relu') elif act_type == 'prelu': self.msAA1 = nn.PReLU() self.msAA2 = nn.PReLU() elif act_type == 'elu': self.msAA1 = nn.ELU() self.msAA2 = nn.ELU() elif act_type == 'selu': self.msAA1 = nn.SELU() self.msAA2 = nn.SELU() elif act_type == 'gelu': self.msAA1 = nn.GELU() self.msAA2 = nn.GELU() elif act_type == 'swish': self.msAA1 = nn.Swish() self.msAA2 = nn.Swish() elif act_type == 'ChaATAC': self.msAA1 = ChaATAC(channels=in_channels, r=r, useReLU=useReLU, useGlobal=useGlobal) self.msAA2 = ChaATAC(channels=channels, r=r, useReLU=useReLU, useGlobal=useGlobal) else: raise ValueError("Unknown act_type in ResBlockV2ATAC")
def test_activations_gelu(): act_layer = nn.GELU() out = act_layer(mx.np.random.uniform(size=(10,))) out.asnumpy()
def __init__(self, units, hidden_size, **kwargs): super(PositionWiseFFN, self).__init__(**kwargs) self.ffn_1 = nn.Dense(hidden_size, flatten=False) self.activation = nn.GELU() self.ffn_2 = nn.Dense(units, flatten=False)
def __init__(self, layers, channels, classes, act_type, r, skernel, dilation, useReLU, useGlobal, act_layers, replace_act, act_order, asBackbone, norm_layer=BatchNorm, norm_kwargs=None, **kwargs): super(ResNet20V2ATAC, self).__init__(**kwargs) assert len(layers) == len(channels) - 1 with self.name_scope(): self.features = nn.HybridSequential(prefix='') self.features.add(norm_layer(scale=False, center=False, **({} if norm_kwargs is None else norm_kwargs))) self.features.add(nn.Conv2D(channels[0], 3, 1, 1, use_bias=False)) in_channels = channels[0] for i, num_layer in enumerate(layers): stride = 1 if i == 0 else 2 if act_order == 'bac': if i + act_layers < len(channels): tmp_act_type = replace_act else: tmp_act_type = act_type elif act_order == 'pre': if i + 1 > act_layers: tmp_act_type = replace_act else: tmp_act_type = act_type else: raise ValueError('Unknown act_order') self.features.add(self._make_layer( layers=num_layer, channels=channels[i+1], in_channels=in_channels, stride=stride, stage_index=i+1, act_type=tmp_act_type, r=r, skernel=skernel, dilation=dilation, useReLU=useReLU, useGlobal=useGlobal, asBackbone=asBackbone, norm_layer=norm_layer, norm_kwargs=norm_kwargs )) in_channels = channels[i+1] self.features.add(norm_layer(**({} if norm_kwargs is None else norm_kwargs))) if act_order == 'bac': if act_layers <= 0: tmp_act_type = replace_act else: tmp_act_type = act_type elif act_order == 'pre': if act_layers >= 4: tmp_act_type = act_type else: tmp_act_type = replace_act else: raise ValueError('Unknown act_order') if tmp_act_type == 'relu': self.features.add(nn.Activation('relu')) elif tmp_act_type == 'prelu': self.features.add(nn.PReLU()) elif tmp_act_type == 'elu': self.features.add(nn.ELU()) elif tmp_act_type == 'selu': self.features.add(nn.SELU()) elif tmp_act_type == 'gelu': self.features.add(nn.GELU()) elif tmp_act_type == 'swish': self.features.add(nn.Swish()) elif tmp_act_type == 'ChaATAC': self.features.add(ChaATAC(channels=in_channels, r=r, useReLU=useReLU, useGlobal=useGlobal)) else: raise ValueError("Unknown act_type in ResBlockV2ATAC") self.features.add(nn.GlobalAvgPool2D()) self.features.add(nn.Flatten()) self.output = nn.Dense(classes, in_units=in_channels)