def __init__(self, indim, outdim): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) WeightNorm.apply( self.L, 'weight', dim=0) #split the weight update component to direction and norm self.relu = nn.ReLU()
def __init__(self, indim, outdim, scale_factor=30): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) self.class_wise_learnable_norm = False # See the issue#4&8 in the github if self.class_wise_learnable_norm: WeightNorm.apply(self.L, 'weight', dim=0) # split the weight update component to direction and norm self.scale_factor = scale_factor
def __init__(self, indim, outdim): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) self.class_wise_learnable_norm = True if self.class_wise_learnable_norm: WeightNorm.apply(self.L, 'weight', dim=0) self.scale_factor = 10
def __init__(self, in_features, out_features): super(CosineClassifier, self).__init__() self.L = nn.Linear(in_features, out_features, bias=False) self.class_wise_learnable_norm = True if self.class_wise_learnable_norm: WeightNorm.apply(self.L, 'weight', dim=0) self.scale_factor = 2
def __init__(self, cfg, feature_shape, in_channel): super().__init__() self.num_classes = cfg.num_classes self.class_mat = nn.Conv2d(in_channel, self.num_classes, 1, bias=False) self.weight_norm = cfg.CLASSIFIER.SEGHEAD.COSINE.weight_norm if self.weight_norm: # conv weight shape: (num_classes, in_channel, 1, 1) WeightNorm.apply(self.class_mat, 'weight', dim=0) self.train_scale_factor = cfg.CLASSIFIER.SEGHEAD.COSINE.train_scale_factor self.val_scale_factor = cfg.CLASSIFIER.SEGHEAD.COSINE.val_scale_factor
def __init__(self, in_features, out_features, temperature): super(MetrixSoftmax, self).__init__() self.in_features = in_features self.out_features = out_features self.temperature = temperature self.weight = Parameter(torch.Tensor(out_features, in_features)) self.register_parameter('bias', None) self.reset_parameters() WeightNorm.apply(self.weight, 'weight', dim=0)
def __init__(self, indim, outdim): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) WeightNorm.apply( self.L, 'weight', dim=0) #split the weight update component to direction and norm if outdim <= 200: self.scale_factor = 2 #a fixed scale factor to scale the output of cos value into a reasonably large input for softmax else: self.scale_factor = 10
def __init__(self, indim, outdim): super(distLinear, self).__init__() self.L = nn.Linear( indim, outdim, bias = False) self.class_wise_learnable_norm = True #See the issue#4&8 in the github if self.class_wise_learnable_norm: WeightNorm.apply(self.L, 'weight', dim=0) #split the weight update component to direction and norm if outdim <=200: self.scale_factor = 2; #a fixed scale factor to scale the output of cos value into a reasonably large input for softmax, for to reproduce the result of CUB with ResNet10, use 4. see the issue#31 in the github else: self.scale_factor = 10; #in omniglot, a larger scale factor is required to handle >1000 output classes.
def __init__(self, indim, outdim, scale_factor=10): super(distLinear, self).__init__() self.indim = indim self.outdim = outdim self.L = nn.Linear( indim, outdim, bias = False) nn.init.xavier_normal_(self.L.weight) self.class_wise_learnable_norm = True #See the issue#4&8 in the github if self.class_wise_learnable_norm: WeightNorm.apply(self.L, 'weight', dim=0) #split the weight update component to direction and norm self.scale_factor = 10; #in omniglot, a larger scale factor is required to handle >1000 output classes.
def __init__(self, indim, outdim): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) self.class_wise_learnable_norm = True # If False, then it is Cosine Classifier, else it is Linear+Crossentropy if self.class_wise_learnable_norm: WeightNorm.apply( self.L, 'weight', dim=0 ) # split the weight update component to direction and norm if outdim <= 200: self.scale_factor = 2 # a fixed scale factor to scale the output of cos value into a reasonably large input for softmax else: self.scale_factor = 10
def maybe_reapply_weight_norm(src: torch.nn.Module, dst: torch.nn.Module) -> torch.nn.Module: #pylint:disable=protected-access for k, hook in src._forward_pre_hooks.items(): if isinstance(hook, WeightNorm): # The code below presumes that the `hook` object does not # contain internal references to the module it was set up on # (i.e. to the `src`) and takes the module to act on as a parameter. # This is the case for the `WeightNorm` hook. hook.remove(dst) del dst._forward_pre_hooks[k] name = hook.name dim = hook.dim WeightNorm.apply(dst, name=name, dim=dim) return dst
def __init__(self, indim, outdim): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) self.class_wise_learnable_norm = True # See the issue#4&8 in the github if self.class_wise_learnable_norm: # split the weight update component to direction and norm WeightNorm.apply(self.L, 'weight', dim=0) if outdim <= 200: # a fixed scale factor to scale the output of cos value into a # reasonably large input for softmax self.scale_factor = 2 else: # in omniglot, a larger scale factor is required to handle >1000 # output classes. self.scale_factor = 10
def __init__(self, indim, outdim, temperature, margin): super(distLinear, self).__init__() self.margin = margin if margin is None: self.L = nn.Linear(indim, outdim, bias=False) else: self.L = LSoftmaxLinear(indim, outdim, margin) self.L.reset_parameters() WeightNorm.apply( self.L, 'weight', dim=0) # split the weight update component to direction and norm if outdim <= 200: self.scale_factor = temperature # a fixed scale factor to scale the output of cos value into a reasonably large input for softmax else: self.scale_factor = temperature # in omniglot, a larger scale factor is required to handle >1000 output classes.
def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.1): super(TemporalBlock, self).__init__() self.conv1 = nn.Conv1d(n_inputs, n_outputs, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=False) WeightNorm.apply(self.conv1, "weight", 0) self.chomp1 = Chomp1d(padding) self.relu1 = nn.ReLU() self.dropout1 = nn.Dropout(dropout) self.conv2 = nn.Conv1d(n_outputs, n_outputs, kernel_size, stride=stride, padding=padding, dilation=dilation, bias=False) WeightNorm.apply(self.conv2, "weight", 0) self.chomp2 = Chomp1d(padding) self.relu2 = nn.ReLU() self.dropout2 = nn.Dropout(dropout) self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1, self.conv2, self.chomp2, self.relu2, self.dropout2) self.downsample = nn.Conv1d( n_inputs, n_outputs, 1, bias=False) if n_inputs != n_outputs else None self.relu = nn.ReLU() self.init_weights()
def __init__(self, indim, outdim, init_orthogonal=False): super(distLinear, self).__init__() self.L = nn.Linear(indim, outdim, bias=False) self.class_wise_learnable_norm = True #See the issue#4&8 in the github if init_orthogonal: self.L.weight.data = torch.from_numpy( generate_orthogonal_rows_matrix( outdim, indim, self.L.weight.data.cpu().numpy())) if self.class_wise_learnable_norm: WeightNorm.apply( self.L, 'weight', dim=0 ) #split the weight update component to direction and norm if outdim <= 200: self.scale_factor = 2 #a fixed scale factor to scale the output of cos value into a reasonably large input for softmax else: self.scale_factor = 10
def __init__(self, n_inputs, n_outputs, kernel_size, stride, dilation, padding, dropout=0.2): super(QTemporalBlock, self).__init__() self.conv1 = QuaternionConv(n_inputs, n_outputs, kernel_size, stride=stride, padding=padding, dilatation=dilation, operation='convolution1d', bias=True) WeightNorm.apply(self.conv1, "r_weight", 0) WeightNorm.apply(self.conv1, "i_weight", 0) WeightNorm.apply(self.conv1, "j_weight", 0) WeightNorm.apply(self.conv1, "k_weight", 0) self.chomp1 = QChomp1d(padding) self.relu1 = nn.PReLU() self.dropout1 = nn.Dropout(dropout) self.conv2 = QuaternionConv(n_outputs, n_outputs, kernel_size, stride=stride, padding=padding, dilatation=dilation, operation='convolution1d', bias=True) WeightNorm.apply(self.conv2, "r_weight", 0) WeightNorm.apply(self.conv2, "i_weight", 0) WeightNorm.apply(self.conv2, "j_weight", 0) WeightNorm.apply(self.conv2, "k_weight", 0) self.chomp2 = QChomp1d(padding) self.relu2 = nn.PReLU() self.dropout2 = nn.Dropout(dropout) self.net = nn.Sequential(self.conv1, self.chomp1, self.relu1, self.dropout1, self.conv2, self.chomp2, self.relu2, self.dropout2) self.downsample = QuaternionConv( n_inputs, n_outputs, 1, stride=1, operation='convolution1d', bias=False) if n_inputs != n_outputs else None self.relu = nn.PReLU()