def __init__(self, act_type, auto_optimize=True, **kwargs): super(Activation, self).__init__() if act_type == 'relu': self.act = nn.ReLU(inplace=True) if auto_optimize else nn.ReLU(**kwargs) elif act_type == 'relu6': self.act = nn.ReLU6(inplace=True) if auto_optimize else nn.ReLU6(**kwargs) elif act_type == 'h_swish': self.act = nn.Hardswish(inplace=True) if auto_optimize else nn.Hardswish(**kwargs) elif act_type == 'h_sigmoid': self.act = nn.Hardsigmoid(inplace=True) if auto_optimize else nn.Hardsigmoid(**kwargs) elif act_type == 'swish': self.act = nn.SiLU(inplace=True) if auto_optimize else nn.SiLU(**kwargs) elif act_type == 'gelu': self.act = nn.GELU() elif act_type == 'elu': self.act = nn.ELU(inplace=True, **kwargs) if auto_optimize else nn.ELU(**kwargs) elif act_type == 'mish': self.act = Mish() elif act_type == 'sigmoid': self.act = nn.Sigmoid() elif act_type == 'lrelu': self.act = nn.LeakyReLU(inplace=True, **kwargs) if auto_optimize else nn.LeakyReLU(**kwargs) elif act_type == 'prelu': self.act = nn.PReLU(**kwargs) else: raise NotImplementedError('{} activation is not implemented.'.format(act_type))
def __init__(self,input_shape,num_feature,num_class,transfered=None): super(VGG16,self).__init__() #load transfered encoder vgg16 = torchvision.models.vgg16_bn(pretrained=False) if transfered: vgg16.load_state_dict(torch.load(transfered)) self.conv = vgg16.features self.fc1 = nn.Sequential( #512*(input_shape[1]/32)*(input_shape[2]/32) nn.Linear(512*(input_shape[1]//32)*(input_shape[2]//32),num_feature) , nn.ReLU() , nn.Dropout(0.5) ) #num_feature self.fc2 = nn.Linear(num_feature,num_class) self.confidnet = nn.Sequential( #num_feature nn.Linear(num_feature,400) , nn.ReLU() , nn.Linear(400,400) , nn.ReLU() , nn.Linear(400,400) , nn.ReLU() , nn.Linear(400,400) , nn.ReLU() , nn.Linear(400,1) , nn.Hardsigmoid() ) #1
def __init__(self, in_channels_x, in_channels_f, num_classes): super(R_ASPP_module, self).__init__() assert not num_classes % 2 self.layer1 = nn.Sequential( nn.Conv2d(in_channels_x, 128, kernel_size=1, stride=1), nn.BatchNorm2d(128), ) self._act = nn.ReLU6() self.avgpool = nn.AdaptiveAvgPool2d(1) self.layer2 = nn.Conv2d(in_channels_x, 128, kernel_size=1, stride=1) self.out_conv1 = nn.Conv2d(128, num_classes // 2, kernel_size=1, stride=1) self.out_conv2 = nn.Conv2d(in_channels_f, num_classes // 2, kernel_size=1, stride=1) self.hsigmoid = nn.Hardsigmoid() self._init_weight()
def __init__(self): super(NNActivationModule, self).__init__() self.activations = nn.ModuleList([ nn.ELU(), nn.Hardshrink(), nn.Hardsigmoid(), nn.Hardtanh(), nn.Hardswish(), nn.LeakyReLU(), nn.LogSigmoid(), # nn.MultiheadAttention(), nn.PReLU(), nn.ReLU(), nn.ReLU6(), nn.RReLU(), nn.SELU(), nn.CELU(), nn.GELU(), nn.Sigmoid(), nn.SiLU(), nn.Mish(), nn.Softplus(), nn.Softshrink(), nn.Softsign(), nn.Tanh(), nn.Tanhshrink(), # nn.Threshold(0.1, 20), nn.GLU(), nn.Softmin(), nn.Softmax(), nn.Softmax2d(), nn.LogSoftmax(), # nn.AdaptiveLogSoftmaxWithLoss(), ])
def __init__(self, window_size=N, hidden=128, embed=32, learning_rate=5e-3): super(agent, self).__init__() self.window_size = window_size self.hidden = hidden self.embed = embed self.learning_rate = learning_rate self.encoder = nn.Sequential( nn.Linear((self.window_size * 2 + 1)**2 * 6, hidden), nn.ReLU(True), nn.Linear(hidden, hidden), nn.ReLU(True), nn.Linear(hidden, embed)) self.decoder = nn.Sequential( nn.Linear(embed, hidden), nn.ReLU(True), nn.Linear(hidden, hidden), nn.ReLU(True), nn.Linear(hidden, (self.window_size * 2 + 1)**2 * 6), nn.Hardsigmoid()) self.optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate, weight_decay=1e-5)
def __init__( self, low_filters: int, high_filters: int, output_filters: Optional[int] = None, kernel_size: Union[int, Tuple[int, ...]] = 3, pool: bool = True, activation: Optional[nn.Module] = nn.Hardsigmoid(), ): super().__init__() output_filters = high_filters if output_filters is None else output_filters kernel_size = self.ToTuple(kernel_size) self.align_corners = False self._pool = pool self.low_level_path = nn.Sequential( DynamicSamePad( self.Conv(low_filters, output_filters, kernel_size, bias=False)), ) # gets weights for low_level self.high_level_path = nn.Sequential( self.AdaptiveAvgPool(1) if pool else nn.Identity(), self.Conv(output_filters, output_filters, 1), deepcopy(activation) if activation is not None else nn.Identity(), ) # pointwise conv when high_filters != output_filters if high_filters != output_filters: self.high_level_conv = self.Conv(high_filters, output_filters, 1) else: self.high_level_conv = nn.Identity()
def __init__(self, k, num_channels, norm=None, group=1, use_hsig=True): super(AttentionWeights, self).__init__() # num_channels *= 2 self.k = k self.avgpool = nn.AdaptiveAvgPool2d(1) self.attention = nn.Sequential( nn.Conv2d(num_channels, k, 1, bias=False), nn.BatchNorm2d(k) if norm == 'BN' else nn.GroupNorm(group, k), nn.Hardsigmoid() if use_hsig else nn.Sigmoid())
def __init__(self, in_c, reduction_ratio=0.25): super(SE, self).__init__() reducation_c = make_divisible(in_c * reduction_ratio, 4) self.block = nn.Sequential( nn.AdaptiveAvgPool2d(1), nn.Conv2d(in_c, reducation_c, kernel_size=1, bias=True), nn.ReLU(inplace=True), nn.Conv2d(reducation_c, in_c, kernel_size=1, bias=True), nn.Hardsigmoid())
def __init__(self, in_channel, ratio=0.25): super(SEBlock, self).__init__() self.hid_dim = max(1, int(in_channel * ratio)) self._pool = nn.AdaptiveAvgPool2d(1) self._ln1 = nn.Linear(in_channel, self.hid_dim) self._ln2 = nn.Linear(self.hid_dim, in_channel) self._act1 = nn.ReLU() self._act2 = nn.Hardsigmoid() # we dont not like sigmoid hihi :^)
def __init__(self, expand_size): super(SqueezeExciteModule, self).__init__() self.se_0_0 = nn.AdaptiveAvgPool2d(output_size=1) self.se_0_1 = nn.Flatten() self.se_1_0 = nn.Linear(in_features=expand_size, out_features=expand_size) self.se_1_1 = nn.ReLU(inplace=True) self.se_2_0 = nn.Linear(in_features=expand_size, out_features=expand_size) self.se_2_1 = nn.Hardsigmoid(inplace=True)
def __init__(self): super(Decoder, self).__init__() self.upconv_bn_relu_1 = nn.Sequential( nn.ConvTranspose2d(64, 32, 4), # 4x4 nn.BatchNorm2d(32), nn.ReLU() ) self.upconv_bn_relu_2 = nn.Sequential( nn.ConvTranspose2d(32, 4, 6, stride=2), # 12x12 nn.BatchNorm2d(32), nn.ReLU() ) self.upconv_3 = nn.ConvTranspose2d(4, 1, 6, stride=2) # 28x28 self.hard_sigmoid = nn.Hardsigmoid()
def build_act(name: Union[str, nn.Module, None]) -> Optional[nn.Module]: if name is None: return None elif isinstance(name, nn.Module): return name elif name == "relu": return nn.ReLU(inplace=True) elif name == "relu6": return nn.ReLU6(inplace=True) elif name == "h_swish": return nn.Hardswish(inplace=True) elif name == "h_sigmoid": return nn.Hardsigmoid(inplace=True) else: raise NotImplementedError
def __init__( self, in_channels: int, squeeze_ratio: float, out_channels: Optional[int] = None, first_activation: nn.Module = nn.ReLU(), second_activation: nn.Module = nn.Hardsigmoid(), global_pool: bool = True, pool_type: Union[str, type] = "avg", ): super().__init__() self.in_channels = abs(int(in_channels)) self.squeeze_ratio = abs(float(squeeze_ratio)) self.out_channels = self.in_channels if out_channels is None else abs( int(out_channels)) mid_channels = int(max(1, in_channels // squeeze_ratio)) # get pooling type from str or provided module type if isinstance(pool_type, str): if pool_type == "avg": self.pool_type = self.AdaptiveAvgPool elif pool_type == "max": self.pool_type = self.AdaptiveMaxPool else: raise ValueError(f"Unknown pool type {pool_type}") elif isinstance(pool_type, type): self.pool_type = pool_type else: raise TypeError( f"Expected str or type for pool_type, found {type(pool_type)}") # for global attention, squeeze uses global pooling and linear layers if global_pool: self.pool = self._get_global_pool() else: self.pool = None self.squeeze = nn.Sequential( self.Conv(self.in_channels, mid_channels, 1), deepcopy(first_activation), ) self.excite = nn.Sequential( self.Conv(mid_channels, self.out_channels, 1), deepcopy(second_activation), )
def make_act(act='ReLU', **kwargs): inplace = kwargs.pop("inplace", True) if len(act) == 0: return None act = { "ReLU": nn.ReLU(inplace=inplace), "ReLU6": nn.ReLU6(inplace=inplace), "PReLU": nn.PReLU(), "LeakyReLU": nn.LeakyReLU(inplace=inplace), "H_Sigmoid": nn.Hardsigmoid(), "Sigmoid": nn.Sigmoid(), "H_Swish": nn.Hardswish(), "Swish": Swish(), "Mish": Mish(), }[act] return act
def activations(self): return { "celu": nn.CELU(), "gelu": nn.GELU(), "lrelu": nn.LeakyReLU(0.2), "prelu": nn.PReLU(), "relu": nn.ReLU(), "relu6": nn.ReLU6(), "selu": nn.SELU(), "sigmoid": nn.Sigmoid(), "softplus": nn.Softplus(), "softshrink": nn.Softshrink(), "softsign": nn.Softsign(), "hardsigmoid": nn.Hardsigmoid(), "hardtanh": nn.Hardtanh(), "tanh": nn.Tanh(), "tanhshrink": nn.Tanhshrink() }
def __init__(self, input_shape, num_feature, num_class, transfered=None): super(ConvNetMNIST, self).__init__() self.conv = nn.Sequential( #input_shape[0]*input_shape[1]*input_shape[2] Conv2dSame(input_shape[0], 32, 3), nn.ReLU(), Conv2dSame(32, 64, 3), nn.ReLU(), nn.MaxPool2d(2), nn.Dropout(0.25)) #64*(input_shape[1]/2)*(input_shape[2]/2) self.fc1 = nn.Sequential( #64*(input_shape[1]/2)*(input_shape[2]/2) nn.Linear(64 * (input_shape[1] // 2) * (input_shape[2] // 2), num_feature), nn.ReLU(), nn.Dropout(0.5)) #num_feature self.fc2 = nn.Linear(num_feature, num_class) self.confidnet = nn.Sequential( #num_feature nn.Linear(num_feature, 400), nn.ReLU(), nn.Linear(400, 400), nn.ReLU(), nn.Linear(400, 400), nn.ReLU(), nn.Linear(400, 400), nn.ReLU(), nn.Linear(400, 1), nn.Hardsigmoid()) #1
def __init__(self, shallow_in_c: int, deep_in_c: int, out_c: int, hidden_c: int = 128): super().__init__() self.deep_conv_1 = ConvBN(deep_in_c, hidden_c, 1, activation=nn.ReLU6()) self.squeeze_excite = nn.Sequential( nn.AdaptiveAvgPool2d((2, 2)), nn.Conv2d(deep_in_c, hidden_c, 1), nn.Hardsigmoid(), ) self.deep_conv_2 = nn.Conv2d(hidden_c, out_c, 1) self.shallow_conv = nn.Conv2d(shallow_in_c, out_c, 1)
def __init__(self, in_dim, out_dim, ks, st, padding=0, norm='bn', activation='relu', pad_type='zero', use_bias=True, activation_first=False): super(Conv2dBlock, self).__init__() self.use_bias = use_bias self.activation_first = activation_first # initialize padding if pad_type == 'reflect': self.pad = nn.ReflectionPad2d(padding) elif pad_type == 'replicate': self.pad = nn.ReplicationPad2d(padding) elif pad_type == 'zero': self.pad = nn.ZeroPad2d(padding) else: assert 0, "Unsupported padding type: {}".format(pad_type) # initialize normalization norm_dim = out_dim if norm == 'bn': self.norm = nn.BatchNorm2d(norm_dim) elif norm == 'in': self.norm = nn.InstanceNorm2d(norm_dim) elif norm == 'none': self.norm = None else: assert 0, "Unsupported normalization: {}".format(norm) # initialize activation if activation == 'relu': self.activation = nn.ReLU(inplace=True) elif activation == 'lrelu': self.activation = nn.LeakyReLU(0.2, inplace=True) elif activation == 'tanh': self.activation = nn.Tanh() elif activation == "hardsigmoid": self.activation = nn.Hardsigmoid() elif activation == "sigmoid": self.activation = nn.Sigmoid() elif activation == 'none': self.activation = None else: assert 0, "Unsupported activation: {}".format(activation) self.conv = nn.Conv2d(in_dim, out_dim, ks, st, bias=self.use_bias)
def make_act(act='ReLU', **kwargs): inplace = kwargs.pop("inplace", True) if len(act) == 0: return None act = { "ReLU": nn.ReLU(inplace=inplace), "ReLU6": nn.ReLU6(inplace=inplace), "PReLU": nn.PReLU(), "LeakyReLU": nn.LeakyReLU(inplace=inplace), "H_Sigmoid": nn.Hardsigmoid(), "Sigmoid": nn.Sigmoid(), "TanH": nn.Tanh(), "H_Swish": nn.Hardswish(), "Swish": ops.Swish(), # torch >= 1.7.0, nn.SiLU() "Mish": ops.Mish(), }[act] return act
def _get_function(name: str, inplace: bool) -> nn.Module: """ Use this to instantiate activations and gating functions by name. """ if name == "relu": return nn.ReLU(inplace=inplace) elif name == "relu6": return nn.ReLU6(inplace=inplace) elif name == "swish": return Swish(inplace=inplace) elif name == "hard_swish": if _HAS_HARDSWISH: return nn.Hardsiwsh() else: return HardSwish(inplace=inplace) elif name == "sigmoid": return Sigmoid(inplace=inplace) elif name == "hard_sigmoid": if _HAS_HARDSWISH: return nn.Hardsigmoid() else: return HardSigmoid(inplace=inplace)
def __init__(self,input_shape,num_feature,num_class,transfered=None): super(ConvNetSVHN2,self).__init__() self.conv = nn.Sequential( #input_shape[0]*input_shape[1]*input_shape[2] Conv2dSame_BN_ReLU(input_shape[0],32,3) , Conv2dSame_BN_ReLU(32,32,3) , nn.MaxPool2d(2) , nn.Dropout(0.3) , Conv2dSame_BN_ReLU(32,64,3) , Conv2dSame_BN_ReLU(64,64,3) , nn.MaxPool2d(2) , nn.Dropout(0.3) , Conv2dSame_BN_ReLU(64,128,3) , Conv2dSame_BN_ReLU(128,128,3) , nn.MaxPool2d(2) , nn.Dropout(0.3) ) #128*(input_shape[1]/8)*(input_shape[2]/8) self.fc1 = nn.Sequential( #128*(input_shape[1]/8)*(input_shape[2]/8) nn.Linear(128*(input_shape[1]//8)*(input_shape[2]//8),num_feature) , nn.ReLU() , nn.Dropout(0.3) ) #num_feature self.fc2 = nn.Linear(num_feature,num_class) self.confidnet = nn.Sequential( #num_feature nn.Linear(num_feature,400) , nn.ReLU() , nn.Linear(400,400) , nn.ReLU() , nn.Linear(400,400) , nn.ReLU() , nn.Linear(400,400) , nn.ReLU() , nn.Linear(400,1) , nn.Hardsigmoid() ) #1
def __init__(self): # 将hard sigmoid 放在 [-1, 1] super(CreateEdges, self).__init__() self.hardsig = nn.Hardsigmoid()
def hsigmoid(inplace=False) -> nn.Module: return nn.Hardsigmoid(inplace=inplace)
def __init__(self): super(Model, self).__init__() self.act_0 = nn.Hardsigmoid()
def __init__(self, add_stub=False): super().__init__() self.quant = QuantStub() self.dequant = DeQuantStub() self.add_stub = add_stub self.hsigmoid = nn.Hardsigmoid()
def __init__( self, d_feat=6, output_dim=1, freq_dim=10, hidden_size=64, dropout_W=0.0, dropout_U=0.0, device="cpu", ): super().__init__() self.input_dim = d_feat self.output_dim = output_dim self.freq_dim = freq_dim self.hidden_dim = hidden_size self.device = device self.W_i = nn.Parameter( init.xavier_uniform_(torch.empty( (self.input_dim, self.hidden_dim)))) self.U_i = nn.Parameter( init.orthogonal_(torch.empty(self.hidden_dim, self.hidden_dim))) self.b_i = nn.Parameter(torch.zeros(self.hidden_dim)) self.W_ste = nn.Parameter( init.xavier_uniform_(torch.empty(self.input_dim, self.hidden_dim))) self.U_ste = nn.Parameter( init.orthogonal_(torch.empty(self.hidden_dim, self.hidden_dim))) self.b_ste = nn.Parameter(torch.ones(self.hidden_dim)) self.W_fre = nn.Parameter( init.xavier_uniform_(torch.empty(self.input_dim, self.freq_dim))) self.U_fre = nn.Parameter( init.orthogonal_(torch.empty(self.hidden_dim, self.freq_dim))) self.b_fre = nn.Parameter(torch.ones(self.freq_dim)) self.W_c = nn.Parameter( init.xavier_uniform_(torch.empty(self.input_dim, self.hidden_dim))) self.U_c = nn.Parameter( init.orthogonal_(torch.empty(self.hidden_dim, self.hidden_dim))) self.b_c = nn.Parameter(torch.zeros(self.hidden_dim)) self.W_o = nn.Parameter( init.xavier_uniform_(torch.empty(self.input_dim, self.hidden_dim))) self.U_o = nn.Parameter( init.orthogonal_(torch.empty(self.hidden_dim, self.hidden_dim))) self.b_o = nn.Parameter(torch.zeros(self.hidden_dim)) self.U_a = nn.Parameter(init.orthogonal_(torch.empty(self.freq_dim, 1))) self.b_a = nn.Parameter(torch.zeros(self.hidden_dim)) self.W_p = nn.Parameter( init.xavier_uniform_(torch.empty(self.hidden_dim, self.output_dim))) self.b_p = nn.Parameter(torch.zeros(self.output_dim)) self.activation = nn.Tanh() self.inner_activation = nn.Hardsigmoid() self.dropout_W, self.dropout_U = (dropout_W, dropout_U) self.fc_out = nn.Linear(self.output_dim, 1) self.states = []
def __init__(self, channels: int, factor: Union[int, float] = 1 / 4): super().__init__() self.squeeze_excite = nn.Sequential( nn.Linear(channels, math.floor(channels * factor)), nn.ReLU(), nn.Linear(math.floor(channels * factor), channels), nn.Hardsigmoid())