Exemplo n.º 1
0
  def __init__(self, block, depth, cardinality, base_width, num_classes):
    super(CifarResNeXt, self).__init__()

    # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
    assert (depth - 2) % 9 == 0, 'depth should be one of 29, 38, 47, 56, 101'
    self.layer_blocks = (depth - 2) // 9

    self.cardinality,self.base_width,self.num_classes,self.block = cardinality,base_width,num_classes,block

    self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False)
    self.bn_1 = nn.BatchNorm2d(64)

    self.inplanes = 64
    self.stage_1 = self._make_layer(64 , 1)
    self.stage_2 = self._make_layer(128, 2)
    self.stage_3 = self._make_layer(256, 2)
    self.avgpool = nn.AdaptiveAvgPool2d((1,1))
    self.classifier = nn.Linear(256*block.expansion, num_classes)

    for m in self.modules():
      if isinstance(m, nn.Conv2d):
        n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
        m.weight.data.normal_(0, math.sqrt(2. / n))
      elif isinstance(m, nn.BatchNorm2d):
        m.weight.data.fill_(1)
        m.bias.data.zero_()
      elif isinstance(m, nn.Linear):
        init.kaiming_normal(m.weight)
        m.bias.data.zero_()
Exemplo n.º 2
0
    def __init__(self, cardinality, depth, num_classes, widen_factor=4, dropRate=0):
        """ Constructor
        Args:
            cardinality: number of convolution groups.
            depth: number of layers.
            num_classes: number of classes
            widen_factor: factor to adjust the channel dimensionality
        """
        super(CifarResNeXt, self).__init__()
        self.cardinality = cardinality
        self.depth = depth
        self.block_depth = (self.depth - 2) // 9
        self.widen_factor = widen_factor
        self.num_classes = num_classes
        self.output_size = 64
        self.stages = [64, 64 * self.widen_factor, 128 * self.widen_factor, 256 * self.widen_factor]

        self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False)
        self.bn_1 = nn.BatchNorm2d(64)
        self.stage_1 = self.block('stage_1', self.stages[0], self.stages[1], 1)
        self.stage_2 = self.block('stage_2', self.stages[1], self.stages[2], 2)
        self.stage_3 = self.block('stage_3', self.stages[2], self.stages[3], 2)
        self.classifier = nn.Linear(1024, num_classes)
        init.kaiming_normal(self.classifier.weight)

        for key in self.state_dict():
            if key.split('.')[-1] == 'weight':
                if 'conv' in key:
                    init.kaiming_normal(self.state_dict()[key], mode='fan_out')
                if 'bn' in key:
                    self.state_dict()[key][...] = 1
            elif key.split('.')[-1] == 'bias':
                self.state_dict()[key][...] = 0
Exemplo n.º 3
0
 def weights_init(m):
     for key in m.state_dict():
         if key.split('.')[-1] == 'weight':
             if 'conv' in key:
                 init.kaiming_normal(m.state_dict()[key], mode='fan_out')
             if 'bn' in key:
                 m.state_dict()[key][...] = 1
         elif key.split('.')[-1] == 'bias':
             m.state_dict()[key][...] = 0
Exemplo n.º 4
0
def weights_init_kaiming(m):
    classname = m.__class__.__name__
    # print(classname)
    if classname.find('Conv') != -1:
        init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
    elif classname.find('Linear') != -1:
        init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
    elif classname.find('BatchNorm2d') != -1:
        init.normal(m.weight.data, 1.0, 0.02)
        init.constant(m.bias.data, 0.0)
Exemplo n.º 5
0
def linear(channel_in, channel_out,
           activation=nn.LeakyReLU,
           normalizer=nn.BatchNorm1d):
    layer = list()
    bias = True if not normalizer else False

    layer.append(nn.Linear(channel_in, channel_out, bias=bias))
    _apply(layer, activation, normalizer, channel_out)
    init.kaiming_normal(layer[0].weight)

    return nn.Sequential(*layer)
Exemplo n.º 6
0
    def __init__(self, depth, pretrained=True, cut_at_pooling=False,
                 num_features=0, norm=False, dropout=0, num_classes=0,
                 num_diff_features=0, iden_pretrain = False,
                 model_path='/media/hh/disc_d/hh/open-reid-master/pretrained model/resnet50.pth'):
        super(ResNet, self).__init__()

        self.depth = depth
        self.pretrained = pretrained
        self.cut_at_pooling = cut_at_pooling
        self.iden_pretrain = iden_pretrain

        # Construct base (pretrained) resnet
        if depth not in ResNet.__factory:
            raise KeyError("Unsupported depth:", depth)
        # self.base = ResNet.__factory[depth](pretrained=pretrained)
        self.base = baseresnet.ResNet(baseresnet.Bottleneck, [3, 4, 6, 3])
        if pretrained is True:
            self.base.load_state_dict(torch.load(model_path))
        self.relu = nn.ReLU(inplace=True)
        if not self.cut_at_pooling:
            self.num_features = num_features
            self.num_diff_features = num_diff_features
            self.norm = norm
            self.dropout = dropout
            self.has_embedding = num_features > 0
            self.num_classes = num_classes

            out_planes = self.base.fc.in_features

            # Append new layers
            if self.has_embedding:
                self.feat = nn.Linear(out_planes, self.num_features)
                self.feat_bn = nn.BatchNorm1d(self.num_features)
                init.kaiming_normal(self.feat.weight, mode='fan_out')
                init.constant(self.feat.bias, 0)
                init.constant(self.feat_bn.weight, 1)
                init.constant(self.feat_bn.bias, 0)
            else:
                # Change the num_features to CNN output channels
                self.num_features = out_planes
            if self.dropout > 0:
                self.drop = nn.Dropout(self.dropout)
            if self.num_diff_features > 0:
                self.diff_feat = nn.Linear(self.num_features, self.num_diff_features)
                init.orthogonal(self.diff_feat.weight)
                init.constant(self.diff_feat.bias, 0)
            if self.num_classes > 0:
                self.classifier = nn.Linear(self.num_features, self.num_classes)
                # init.orthogonal(self.classifier.weight)
                init.normal(self.classifier.weight, std=0.001)
                init.constant(self.classifier.bias, 0)

        if not self.pretrained:
            self.reset_params()
Exemplo n.º 7
0
 def reset_params(self):
     for m in self.modules():
         if isinstance(m, nn.Conv2d):
             init.kaiming_normal(m.weight, mode='fan_out')
             if m.bias is not None:
                 init.constant(m.bias, 0)
         elif isinstance(m, nn.BatchNorm2d):
             init.constant(m.weight, 1)
             init.constant(m.bias, 0)
         elif isinstance(m, nn.Linear):
             init.normal(m.weight, std=0.001)
             if m.bias is not None:
                 init.constant(m.bias, 0)
Exemplo n.º 8
0
def init_params(net):
    '''Init layer parameters.'''
    for m in net.modules():
        if isinstance(m, nn.Conv2d):
            init.kaiming_normal(m.weight, mode='fan_out')
            if m.bias:
                init.constant(m.bias, 0)
        elif isinstance(m, nn.BatchNorm2d):
            init.constant(m.weight, 1)
            init.constant(m.bias, 0)
        elif isinstance(m, nn.Linear):
            init.normal(m.weight, std=1e-3)
            if m.bias:
                init.constant(m.bias, 0)
Exemplo n.º 9
0
def conv_transpose2d(channel_in, channel_out,
                     ksize=4, stride=2, padding=1,
                     activation=nn.LeakyReLU,
                     normalizer=nn.BatchNorm2d):
    layer = list()
    bias = True if not normalizer else False

    layer.append(nn.ConvTranspose2d(channel_in, channel_out,
                              ksize, stride, padding,
                              bias=bias))
    _apply(layer, activation, normalizer, channel_out)
    init.kaiming_normal(layer[0].weight)

    return nn.Sequential(*layer)
Exemplo n.º 10
0
def nn_conv2d(channel_in, channel_out,
              ksize=3, stride=1, padding=1,
              scale_factor=2,
              activation=nn.LeakyReLU,
              normalizer=nn.BatchNorm2d):
    layer = list()
    bias = True if not normalizer else False

    layer.append(nn.UpsamplingNearest2d(scale_factor=scale_factor))
    layer.append(nn.Conv2d(channel_in, channel_out,
                           ksize, stride, padding,
                           bias=bias))
    _apply(layer, activation, normalizer, channel_out)
    init.kaiming_normal(layer[1].weight)

    return nn.Sequential(*layer)
Exemplo n.º 11
0
Arquivo: utils.py Projeto: phonx/MUNIT
 def init_fun(m):
     classname = m.__class__.__name__
     if (classname.find('Conv') == 0 or classname.find('Linear') == 0) and hasattr(m, 'weight'):
         # print m.__class__.__name__
         if init_type == 'gaussian':
             init.normal(m.weight.data, 0.0, 0.02)
         elif init_type == 'xavier':
             init.xavier_normal(m.weight.data, gain=math.sqrt(2))
         elif init_type == 'kaiming':
             init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
         elif init_type == 'orthogonal':
             init.orthogonal(m.weight.data, gain=math.sqrt(2))
         elif init_type == 'default':
             pass
         else:
             assert 0, "Unsupported initialization: {}".format(init_type)
         if hasattr(m, 'bias') and m.bias is not None:
             init.constant(m.bias.data, 0.0)
 def init_func(m):
     classname = m.__class__.__name__
     if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
         if init_type == 'normal':
             init.normal(m.weight.data, 0.0, gain)
         elif init_type == 'xavier':
             init.xavier_normal(m.weight.data, gain=gain)
         elif init_type == 'kaiming':
             init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
         elif init_type == 'orthogonal':
             init.orthogonal(m.weight.data, gain=gain)
         else:
             raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
         if hasattr(m, 'bias') and m.bias is not None:
             init.constant(m.bias.data, 0.0)
     elif classname.find('BatchNorm2d') != -1:
         init.normal(m.weight.data, 1.0, gain)
         init.constant(m.bias.data, 0.0)
Exemplo n.º 13
0
    def __init__(self, num_features=0):
        super(Reconstruct, self).__init__()
        self.num_features = num_features

        self.fc_re = nn.Linear(self.num_features, 2048)
        self.fc_rebn = nn.BatchNorm1d(2048)
        init.kaiming_normal(self.fc_re.weight, mode='fan_out')
        init.constant(self.fc_re.bias, 0)
        # init.constant(self.fc_rebn.weight, 1)
        # init.constant(self.fc_rebn.bias, 0)
        self.upconv5 = nn.ConvTranspose2d(2048, 1024, 3, 2, 1)
        self.upconv5_bn = nn.BatchNorm2d(1024)
        self.upconv4 = nn.ConvTranspose2d(1024, 512, 3, 2, 1)
        self.upconv4_bn = nn.BatchNorm2d(512)
        self.upconv3 = nn.ConvTranspose2d(512, 256, 3, 2, 1)
        self.upconv3_bn = nn.BatchNorm2d(256)
        self.upconv2 = nn.ConvTranspose2d(256, 64, 3, 1, 1)
        self.upconv2_bn = nn.BatchNorm2d(64)
        self.upconv1 = nn.ConvTranspose2d(64, 3, 7, 2, 3)
Exemplo n.º 14
0
    def __init__(self, depth, pretrained=True, cut_at_pooling=False,
                 num_features=0, norm=False, dropout=0, num_classes=0):
        super(ResNet, self).__init__()

        self.depth = depth
        self.pretrained = pretrained
        self.cut_at_pooling = cut_at_pooling

        # Construct base (pretrained) resnet
        if depth not in ResNet.__factory:
            raise KeyError("Unsupported depth:", depth)
        self.base = ResNet.__factory[depth](pretrained=pretrained)

        if not self.cut_at_pooling:
            self.num_features = num_features
            self.norm = norm
            self.dropout = dropout
            self.has_embedding = num_features > 0
            self.num_classes = num_classes

            out_planes = self.base.fc.in_features

            # Append new layers
            if self.has_embedding:
                self.feat = nn.Linear(out_planes, self.num_features)
                self.feat_bn = nn.BatchNorm1d(self.num_features)
                init.kaiming_normal(self.feat.weight, mode='fan_out')
                init.constant(self.feat.bias, 0)
                init.constant(self.feat_bn.weight, 1)
                init.constant(self.feat_bn.bias, 0)
            else:
                # Change the num_features to CNN output channels
                self.num_features = out_planes
            if self.dropout > 0:
                self.drop = nn.Dropout(self.dropout)
            if self.num_classes > 0:
                self.classifier = nn.Linear(self.num_features, self.num_classes)
                init.normal(self.classifier.weight, std=0.001)
                init.constant(self.classifier.bias, 0)

        if not self.pretrained:
            self.reset_params()
 def __init__(self, margin=0, num_classes=128):
     super(AdaptTripletLoss, self).__init__()
     self.margin = margin
     self.ranking_loss = nn.MarginRankingLoss(margin=margin)
     self.softmargin_loss = nn.SoftMarginLoss()
     self.num_classes = num_classes
     self.adp_metric_embedding1 = nn.Linear(2*self.num_classes, self.num_classes,bias=False)
     # self.adp_metric_embedding1_bn = nn.BatchNorm1d(self.num_classes)
     self.adp_metric_embedding2 = nn.Linear(self.num_classes, self.num_classes,bias=False)
     # self.adp_metric_embedding2_bn = nn.BatchNorm1d(self.num_classes)
     # init.constant(self.adp_metric_embedding1.bias,0)
     # init.constant(self.adp_metric_embedding2.bias,0)
     init.kaiming_normal(self.adp_metric_embedding1.weight, mode='fan_out')
     init.kaiming_normal(self.adp_metric_embedding2.weight, mode='fan_out')
     self.transform_matrix = nn.Parameter(torch.Tensor(self.num_classes, self.num_classes))
     init.kaiming_normal(self.transform_matrix, mode='fan_out')
Exemplo n.º 16
0
 def __init__(self, margin=0, num_feature=128):
     super(AdaptTripletLoss, self).__init__()
     self.margin = margin
     self.ranking_loss = nn.MarginRankingLoss(margin=margin)
     self.softmargin_loss = nn.SoftMarginLoss()
     self.num_classes = num_feature
     self.adp_metric_embedding1 = nn.Linear(3*self.num_classes, 3*self.num_classes, bias=False)
     self.adp_metric_embedding1_bn = nn.BatchNorm1d(3*self.num_classes)
     self.adp_metric_embedding2 = nn.Linear(3*self.num_classes, 2*self.num_classes, bias=False)
     self.adp_metric_embedding2_bn = nn.BatchNorm1d(2 * self.num_classes)
     self.adp_metric_embedding3 = nn.Linear(2*self.num_classes, 2*self.num_classes, bias=False)
     self.adp_metric_embedding3_bn = nn.BatchNorm1d(2 * self.num_classes)
     self.adp_metric_embedding4 = nn.Linear(2*self.num_classes, 2*self.num_classes, bias=False)
     # self.adp_metric_embedding2_bn = nn.BatchNorm1d(self.num_classes)
     # init.constant(self.adp_metric_embedding1.bias,0)
     # init.constant(self.adp_metric_embedding2.bias,0)
     init.kaiming_normal(self.adp_metric_embedding1.weight, mode='fan_out')
     init.kaiming_normal(self.adp_metric_embedding2.weight, mode='fan_out')
     init.kaiming_normal(self.adp_metric_embedding3.weight, mode='fan_out')
     init.orthogonal(self.adp_metric_embedding4.weight)
Exemplo n.º 17
0
def conv_params(ni, no, k=1):
    return cast(kaiming_normal(torch.Tensor(no, ni, k, k)))
Exemplo n.º 18
0
    def __init__(self,
                 block,
                 depth,
                 num_classes,
                 input_grain_size=(1, 1),
                 input_num_bits=4,
                 input_M2D=0.0,
                 res_grain_size=(1, 1),
                 res_num_bits=4,
                 res_M2D=0.0,
                 output_grain_size=(1, 1),
                 output_num_bits=4,
                 output_M2D=0.0,
                 save_path='./'):
        """ Constructor
    Args:
      depth: number of layers.
      num_classes: number of classes
      grain: grain size as tuple
      M2D: Mean to Deviation ratio
      base_width: base width
    """
        super(CifarResNet, self).__init__()

        #Model type specifies number of layers for CIFAR-10 and CIFAR-100 model
        assert (depth -
                2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
        layer_blocks = (depth - 2) // 6
        print('CifarResNet : Depth : {} , Layers for each block : {}'.format(
            depth, layer_blocks))

        self.num_classes = num_classes

        self.conv_1_3x3 = quan_Conv2d(3,
                                      16,
                                      kernel_size=3,
                                      stride=1,
                                      padding=1,
                                      bias=False,
                                      grain_size=input_grain_size,
                                      num_bits=input_num_bits,
                                      M2D=input_M2D,
                                      save_path=save_path)

        self.inplanes = 16
        self.stage_1 = self._make_layer(block,
                                        16,
                                        layer_blocks,
                                        1,
                                        res_grain_size,
                                        res_num_bits,
                                        res_M2D,
                                        save_path=save_path)
        self.stage_2 = self._make_layer(block,
                                        32,
                                        layer_blocks,
                                        2,
                                        res_grain_size,
                                        res_num_bits,
                                        res_M2D,
                                        save_path=save_path)
        self.stage_3 = self._make_layer(block,
                                        64,
                                        layer_blocks,
                                        2,
                                        res_grain_size,
                                        res_num_bits,
                                        res_M2D,
                                        save_path=save_path)
        self.avgpool = nn.AvgPool2d(8)
        self.classifier = quan_Linear(64 * block.expansion,
                                      num_classes,
                                      output_grain_size,
                                      output_num_bits,
                                      output_M2D,
                                      save_path=save_path)

        for m in self.modules():
            if isinstance(m, nn.Conv2d):
                n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                m.weight.data.normal_(0, math.sqrt(2. / n))
                #m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
            elif isinstance(m, nn.Linear):
                init.kaiming_normal(m.weight)
                m.bias.data.zero_()
Exemplo n.º 19
0
def linear_init(ni, no):
    return kaiming_normal(torch.Tensor(no, ni))
Exemplo n.º 20
0
    def _construct(self, layer_config):
        """
        Method to construct the layer from the layer_config dictionary parameters.
        """
        self.linear = nn.Linear(layer_config['n_in'], layer_config['n_out'])
        self.bn = lambda x: x
        if 'batch_norm' in layer_config:
            if layer_config['batch_norm']:
                self.bn = nn.BatchNorm1d(layer_config['n_out'], momentum=0.99)
        if 'weight_norm' in layer_config:
            if layer_config['weight_norm']:
                self.linear = nn.utils.weight_norm(self.linear, name='weight')

        init_gain = 1.
        if 'non_linearity' in layer_config:
            non_linearity = layer_config['non_linearity']
            if non_linearity is None or non_linearity == 'linear':
                self.non_linearity = lambda x: x
            elif non_linearity == 'relu':
                self.non_linearity = nn.ReLU()
                init_gain = init.calculate_gain('relu')
            elif non_linearity == 'leaky_relu':
                self.non_linearity = nn.LeakyReLU()
            elif non_linearity == 'clipped_leaky_relu':
                self.non_linearity = ClippedLeakyReLU(negative_slope=1. / 3,
                                                      clip_min=-3,
                                                      clip_max=3)
            elif non_linearity == 'elu':
                self.non_linearity = nn.ELU()
            elif non_linearity == 'selu':
                self.non_linearity = nn.SELU()
            elif non_linearity == 'tanh':
                self.non_linearity = nn.Tanh()
                init_gain = init.calculate_gain('tanh')
            elif non_linearity == 'sigmoid':
                self.non_linearity = nn.Sigmoid()
            else:
                raise Exception('Non-linearity ' + str(non_linearity) +
                                ' not found.')
        else:
            self.non_linearity = lambda x: x

        self.dropout = lambda x: x
        if 'dropout' in layer_config:
            if layer_config['dropout'] is not None:
                self.dropout = nn.Dropout(layer_config['dropout'])

        if 'initialize' in layer_config:
            initialize = layer_config['initialize']
            if initialize == 'normal':
                init.normal(self.linear.weight)
            elif initialize == 'glorot_uniform':
                init.xavier_uniform(self.linear.weight, gain=init_gain)
            elif initialize == 'glorot_normal':
                init.xavier_normal(self.linear.weight, gain=init_gain)
            elif initialize == 'kaiming_uniform':
                init.kaiming_uniform(self.linear.weight)
            elif initialize == 'kaiming_normal':
                init.kaiming_normal(self.linear.weight)
            elif initialize == 'orthogonal':
                init.orthogonal(self.linear.weight, gain=init_gain)
            elif initialize == '':
                pass
            else:
                raise Exception('Parameter initialization ' + str(initialize) +
                                ' not found.')

            if 'batch_norm' in layer_config:
                if layer_config['batch_norm']:
                    init.normal(self.bn.weight, 1, 0.02)
                    init.constant(self.bn.bias, 0.)
        else:
            init.xavier_normal(self.linear.weight, gain=init_gain)

        init.constant(self.linear.bias, 0.)
Exemplo n.º 21
0
    def __init__(self,
                 depth,
                 pretrained=True,
                 cut_at_pooling=False,
                 num_features=0,
                 norm=False,
                 dropout=0,
                 num_classes=0,
                 FCN=False,
                 T=1,
                 dim=256):
        super(ResNet, self).__init__()

        self.depth = depth
        self.pretrained = pretrained
        self.cut_at_pooling = cut_at_pooling
        self.FCN = FCN
        self.T = T
        self.reduce_dim = dim
        #        self.offset = ConvOffset2D(32)
        # Construct base (pretrained) resnet
        if depth not in ResNet.__factory:
            raise KeyError("Unsupported depth:", depth)
        self.base = ResNet.__factory[depth](pretrained=pretrained)

        #==========================add dilation=============================#
        if self.FCN:
            self.base.layer4[0].conv2.stride = (1, 1)
            #            self.base.layer4[0].conv2.dilation=(2,2)
            #            self.base.layer4[0].conv2.padding=(2,2)
            self.base.layer4[0].downsample[0].stride = (1, 1)
            #            self.base.layer4[1].conv2.dilation=(2,2)
            #            self.base.layer4[1].conv2.padding=(2,2)
            #            self.base.layer4[2].conv2.dilation=(2,2)
            #            self.base.layer4[2].conv2.padding=(2,2)
            #================append conv for FCN==============================#
            self.num_features = num_features
            self.num_classes = num_classes - 1
            self.dropout = dropout
            self.local_conv = nn.Conv2d(2048,
                                        self.num_features,
                                        kernel_size=1,
                                        padding=0,
                                        bias=False)
            init.kaiming_normal(self.local_conv.weight, mode='fan_out')
            #            init.constant(self.local_conv.bias,0)
            self.feat_bn2d = nn.BatchNorm2d(
                self.num_features)  #may not be used, not working on caffe
            init.constant(self.feat_bn2d.weight,
                          1)  #initialize BN, may not be used
            init.constant(self.feat_bn2d.bias,
                          0)  # iniitialize BN, may not be used

            ##---------------------------stripe1----------------------------------------------#
            self.instance0 = nn.Linear(self.num_features, self.num_classes)
            init.normal(self.instance0.weight, std=0.001)
            init.constant(self.instance0.bias, 0)
            ##---------------------------stripe1----------------------------------------------#
            ##---------------------------stripe1----------------------------------------------#
            self.instance1 = nn.Linear(self.num_features, self.num_classes)
            init.normal(self.instance1.weight, std=0.001)
            init.constant(self.instance1.bias, 0)
            ##---------------------------stripe1----------------------------------------------#
            ##---------------------------stripe1----------------------------------------------#
            self.instance2 = nn.Linear(self.num_features, self.num_classes)
            init.normal(self.instance2.weight, std=0.001)
            init.constant(self.instance2.bias, 0)
            ##---------------------------stripe1----------------------------------------------#
            ##---------------------------stripe1----------------------------------------------#
            self.instance3 = nn.Linear(self.num_features, self.num_classes)
            init.normal(self.instance3.weight, std=0.001)
            init.constant(self.instance3.bias, 0)
            ##---------------------------stripe1----------------------------------------------#
            ##---------------------------stripe1----------------------------------------------#
            self.instance4 = nn.Linear(self.num_features, self.num_classes)
            init.normal(self.instance4.weight, std=0.001)
            init.constant(self.instance4.bias, 0)
            ##---------------------------stripe1----------------------------------------------#
            ##---------------------------stripe1----------------------------------------------#
            self.instance5 = nn.Linear(self.num_features, self.num_classes)
            init.normal(self.instance5.weight, std=0.001)
            init.constant(self.instance5.bias, 0)
            ##---------------------------stripe1----------------------------------------------#
            ##---------------------------stripe1----------------------------------------------#

            self.drop = nn.Dropout(self.dropout)
            self.local_mask = nn.Conv2d(self.reduce_dim,
                                        6,
                                        kernel_size=1,
                                        padding=0,
                                        bias=True)
            init.kaiming_normal(self.local_mask.weight, mode='fan_out')
            #            init.xavier_normal(self.local_mask.weight)
            init.constant(self.local_mask.bias, 0)

#===================================================================#

        elif not self.cut_at_pooling:
            self.num_features = num_features
            self.norm = norm
            self.dropout = dropout
            self.has_embedding = num_features > 0
            self.num_classes = num_classes

            out_planes = self.base.fc.in_features

            # Append new layers
            if self.has_embedding:
                #                self.f_bn = nn.BatchNorm1d(2048)
                #                init.constant(self.f_bn.weight, 1)
                #                init.constant(self.f_bn.bias, 0)

                self.feat = nn.Linear(out_planes,
                                      self.num_features,
                                      bias=False)
                self.feat_bn = nn.BatchNorm1d(self.num_features)
                init.kaiming_normal(self.feat.weight, mode='fan_out')
                #                init.constant(self.feat.bias, 0)
                init.constant(self.feat_bn.weight, 1)
                init.constant(self.feat_bn.bias, 0)
            else:
                # Change the num_features to CNN output channels
                self.num_features = out_planes
            if self.dropout > 0:
                self.drop = nn.Dropout(self.dropout)
            if self.num_classes > 0:
                #                self.classifier = nn.Linear(self.num_features, self.num_classes)
                self.classifier = nn.Linear(self.num_features,
                                            self.num_classes)
                init.normal(self.classifier.weight, std=0.001)
                init.constant(self.classifier.bias, 0)

        if not self.pretrained:
            self.reset_params()
Exemplo n.º 22
0
    def __init__(self, input_channel=7):
        super(NormNetDF, self).__init__()

        self.input_channel = input_channel
        self.res_scale = 7  # number of residuals

        # improved with shrink res-block layers
        in_planes = input_channel
        self.conv1 = conv(in_planes, 64, 7, 2)
        self.conv2 = ResBlock(64, 128, 2)
        self.conv3 = ResBlock(128, 256, 2)
        self.conv3_1 = ResBlock(256, 256)
        self.conv4 = ResBlock(256, 512, stride=2)
        self.conv4_1 = ResBlock(512, 512)
        self.conv5 = ResBlock(512, 512, stride=2)
        self.conv5_1 = ResBlock(512, 512)
        self.conv6 = ResBlock(512, 1024, stride=2)
        self.conv6_1 = ResBlock(1024, 1024)

        # original shrink conv layers
        #self.conv2   = conv(self.batchNorm, 64, 128, 5, 2)
        #self.conv3   = conv(self.batchNorm, 128, 256, 5, 2)
        #self.conv3_1 = conv(self.batchNorm, 256, 256)
        #self.conv4   = conv(self.batchNorm, 256, 512, stride=2)
        #self.conv4_1 = conv(self.batchNorm, 512, 512)
        #self.conv5   = conv(self.batchNorm, 512, 512, stride=2)
        #self.conv5_1 = conv(self.batchNorm, 512, 512)
        #self.conv6   = conv(self.batchNorm, 512, 1024, stride=2)
        #self.conv6_1 = conv(self.batchNorm, 1024, 1024)

        # iconv with deconv layers
        '''
        self.iconv5 = nn.ConvTranspose2d(1025, 512, 3, 1, 1)
        self.iconv4 = nn.ConvTranspose2d(769, 256, 3, 1, 1)
        self.iconv3 = nn.ConvTranspose2d(385, 128, 3, 1, 1)
        self.iconv2 = nn.ConvTranspose2d(193, 64, 3, 1, 1)
        self.iconv1 = nn.ConvTranspose2d(97, 32, 3, 1, 1)
        self.iconv0 = nn.ConvTranspose2d(17+self.input_channel, 16, 3, 1, 1)
        '''
        self.iconv5 = nn.ConvTranspose2d(512 * 3, 512, 3, 1, 1)
        self.iconv4 = nn.ConvTranspose2d(256 + 512 + 256, 256, 3, 1, 1)
        self.iconv3 = nn.ConvTranspose2d(128 + 256 + 128, 128, 3, 1, 1)
        self.iconv2 = nn.ConvTranspose2d(64 + 128 + 64, 64, 3, 1, 1)
        self.iconv1 = nn.ConvTranspose2d(32 + 64 + 32, 32, 3, 1, 1)
        self.iconv0 = nn.ConvTranspose2d(16 + 3 + 16, 16, 3, 1, 1)

        # expand and produce disparity
        self.upconv5 = deconv(1024, 512)
        self.upconv4 = deconv(512, 256)
        self.upconv3 = deconv(256, 128)
        self.upconv2 = deconv(128, 64)
        self.upconv1 = deconv(64, 32)
        self.upconv0 = deconv(32, 16)
        self.pred_res0 = predict_flow(16, 3)

        # weight initialization
        for m in self.modules():
            if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
                # n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
                # m.weight.data.normal_(0, 0.02 / n)
                # m.weight.data.normal_(0, 0.02)
                kaiming_normal(m.weight.data)
                if m.bias is not None:
                    m.bias.data.zero_()
            elif isinstance(m, nn.BatchNorm2d):
                m.weight.data.fill_(1)
                m.bias.data.zero_()
def _weights_init(m):
    classname = m.__class__.__name__
    # print(classname)
    if isinstance(m, MetaLinear) or isinstance(m, MetaConv2d):
        init.kaiming_normal(m.weight)
 def init_weights( self ):
     '''
     Initialize final classifier weights
     '''
     init.kaiming_normal( self.mlp.weight, mode='fan_in' )
     self.mlp.bias.data.fill_( 0 )
    googlenet.load_state_dict(torch.load(opt.googlenet))

#freeze the parameters
#for param in googlenet.parameters():
#param.requires_grad = False
#print(dir(googlenet.module))
#num_ftrs = googlenet.module.classifier.in_features
#googlenet.module.classifier = AngleLinear(num_ftrs,opt.out_class)
googlenet = googlenet.module
for param in googlenet.parameters():
    param.requires_grad = False
#os.environ["CUDA_VISIBLE_DEVICES"] = "5"
num_ftrs = googlenet.classifier.in_features
#googlenet.classifier = AngleLinear(num_ftrs,opt.out_class,opt.m)
googlenet.classifier = nn.Linear(num_ftrs, opt.out_class)
init.kaiming_normal(googlenet.classifier.weight.data, a=0, mode='fan_in')
if ngpu > 1:
    googlenet = nn.DataParallel(googlenet)
#print('m={}'.format(opt.m))

if opt.cuda:
    googlenet.cuda()
    criterion.cuda()
#if ngpu>1:
#googlenet.module.classifier = nn.DataParallel(googlenet.module.classifier)

#print(dir(googlenet.module))
#exit(0)
#googlenet.classifier = nn.Linear(num_ftrs,opt.out_class)
#init.kaiming_normal(googlenet.classifier.weight.data, a=0, mode='fan_in')
Exemplo n.º 26
0
 def reset_parameters(self):
     kaiming_normal(self.weight)
     ZeroInitializer(self.b1)
     ZeroInitializer(self.b2)
Exemplo n.º 27
0
    def __init__(self,
                 growth_rate=32,
                 block_config=(6, 12, 24, 16),
                 num_init_features=64,
                 bn_size=4,
                 drop_rate=0,
                 hidden_num=2048,
                 cut_at_pooling=False,
                 num_features=0,
                 norm=False,
                 dropout=0,
                 num_classes=0):

        super(DenseNet, self).__init__()
        self.cut_at_pooling = cut_at_pooling
        # First convolution
        self.features = nn.Sequential(
            OrderedDict([
                ('conv0',
                 nn.Conv2d(3,
                           num_init_features,
                           kernel_size=7,
                           stride=2,
                           padding=3,
                           bias=False)),
                ('norm0', nn.BatchNorm2d(num_init_features)),
                ('relu0', nn.ReLU(inplace=True)),
                ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
            ]))

        # Each denseblock
        num_features = num_init_features
        for i, num_layers in enumerate(block_config):
            if i < 2:
                block = _DenseBlock(num_layers=num_layers,
                                    num_input_features=num_features,
                                    bn_size=bn_size,
                                    growth_rate=growth_rate,
                                    drop_rate=drop_rate)
                self.features.add_module('denseblock%d' % (i + 1), block)
                num_features = num_features + num_layers * growth_rate
                if i != len(block_config) - 1:
                    trans = _Transition(num_input_features=num_features,
                                        num_output_features=num_features // 2)
                    self.features.add_module('transition%d' % (i + 1), trans)
                    num_features = num_features // 2
        self.hidden = nn.Sequential(
            OrderedDict([
                ('norm2', nn.BatchNorm2d(num_features)),
                ('relu2', nn.ReLU(inplace=True)),
                ('conv3',
                 nn.Conv2d(num_features,
                           num_features,
                           kernel_size=3,
                           stride=1,
                           padding=1,
                           bias=False)),
                ('norm3', nn.BatchNorm2d(num_features)),
                ('relu3', nn.ReLU(inplace=True)),
                ('pool3', nn.MaxPool2d(kernel_size=2, stride=2, padding=1)),
                ('conv4',
                 nn.Conv2d(num_features,
                           num_features,
                           kernel_size=3,
                           stride=1,
                           padding=1,
                           bias=False)),
                ('norm4', nn.BatchNorm2d(num_features)),
                ('relu4', nn.ReLU(inplace=True)),
                ('pool4', nn.MaxPool2d(kernel_size=2, stride=2, padding=1)),
            ]))
        self.embedding = nn.Sequential(
            OrderedDict([('feat', nn.Linear(3 * 6 * 512, hidden_num)),
                         ('feat_bn', nn.BatchNorm2d(hidden_num)),
                         ('feat_relu', nn.ReLU(inplace=True))]))
        if not self.cut_at_pooling:
            self.num_features = num_features
            self.norm = norm
            self.dropout = dropout
            self.has_embedding = num_features > 0
            self.num_classes = num_classes

            out_planes = self.base.classifier.in_features

            # Append new layers
            if self.has_embedding:
                self.feat = nn.Linear(out_planes, self.num_features)
                self.feat_bn = nn.BatchNorm1d(self.num_features)
                init.kaiming_normal(self.feat.weight, mode='fan_out')
                init.constant(self.feat.bias, 0)
                init.constant(self.feat_bn.weight, 1)
                init.constant(self.feat_bn.bias, 0)
            else:
                # Change the num_features to CNN output channels
                self.num_features = out_planes
            if self.dropout > 0:
                self.drop = nn.Dropout(self.dropout)
            if self.num_classes > 0:
                self.classifier_bn = nn.BatchNorm1d(self.num_features)
                self.classifier = nn.Linear(self.num_features,
                                            self.num_classes)
                init.constant(self.classifier_bn.weight, 1)
                init.constant(self.classifier_bn.bias, 0)
                init.normal(self.classifier.weight, std=0.001)
                init.constant(self.classifier.bias, 0)
Exemplo n.º 28
0
def weights_init_kaiming(m):
    classname = m.__class__.__name__
    if classname.find('Conv2d') != -1:
        init.kaiming_normal(m.weight.data)
Exemplo n.º 29
0
        if Deep4:
            # final_conv_length determines the size of the receptive field of the ConvNet
            model = Deep4Net(in_chans=in_chans,
                             n_classes=1,
                             input_time_length=input_time_length,
                             pool_time_stride=pool_time_stride,
                             final_conv_length=2,
                             stride_before_pool=True).create_network()
        elif ResNet:
            model_name = 'resnet-xavier-uniform'
            init_name = model_name.lstrip('resnet-')
            from torch.nn import init
            init_fn = {
                'he-uniform': lambda w: init.kaiming_uniform(w, a=0),
                'he-normal': lambda w: init.kaiming_normal(w, a=0),
                'xavier-uniform': lambda w: init.xavier_uniform(w, gain=1),
                'xavier-normal': lambda w: init.xavier_normal(w, gain=1)
            }[init_name]

            model = EEGResNet(in_chans=in_chans,
                              n_classes=1,
                              input_time_length=input_time_length,
                              final_pool_length=2,
                              n_first_filters=48,
                              conv_weight_init_fn=init_fn).create_network()
        elif EEGNet_v4:
            model = EEGNetv4(
                in_chans=in_chans,
                n_classes=1,
                final_conv_length=2,
Exemplo n.º 30
0
	def KaimingInit(weight, activation):
		assert not activation is None
		if hasattr(activation, "negative_slope"):
			kaiming_normal(weight, a = activation.negative_slope)
		else:
			kaiming_normal(weight, a = 0)
Exemplo n.º 31
0
class MNISTResNet(ResNet):
    def __init__(self):
        super(MNISTResNet, self).__init__(BasicBlock, [2, 2, 2, 2],
                                          num_classes=10)
        self.conv1 = nn.Conv2d(1,
                               64,
                               kernel_size=3,
                               stride=1,
                               padding=3,
                               bias=False)


if __name__ == "__main__":
    model = CNN()

    new_model = nn.Sequential(*list(model.children()))

    conv_model = nn.Sequential()
    for layer in model.named_modules():
        if isinstance(layer[1], nn.Conv2d):
            conv_model.add_module(layer[0], layer[1])

    for m in model.modules():
        if isinstance(m, nn.Conv2d):
            init.normal(m.weight.data)
            init.xavier_normal(m.weight.data)
            init.kaiming_normal(m.weight.data)
            m.bias.data.fill_(0)
        elif isinstance(m, nn.Linear):
            m.weight.data.normal_()
Exemplo n.º 32
0
def _weights_init(m):
    if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):
        init.kaiming_normal(m.weight)
Exemplo n.º 33
0
def linear_params(ni, no):
    return cast({'weight': kaiming_normal(torch.Tensor(no, ni)), 'bias': torch.zeros(no)})
    def __init__(self, args):
        self.use_cuda = args.cuda and torch.cuda.is_available()
        self.args = args

        self.encoder = args.encoder
        self.decoder = args.decoder
        self.z_dim_bern = args.z_dim_bern
        self.z_dim_gauss = args.z_dim_gauss
        self.AE = args.AE

        self.n_filter = args.n_filter
        self.n_rep = args.n_rep
        self.kernel_size = args.kernel_size
        self.padding = args.padding
        self.sbd = args.sbd
        self.freeze_decoder = args.freeze_decoder

        self.n_digits = 2

        if args.dataset.lower() == 'digits_gray':
            self.nc = 1
        elif args.dataset.lower() == 'digits_col':
            self.nc = 3
        else:
            raise NotImplementedError

        net = multi_VAE(self.encoder, self.decoder, self.z_dim_bern,
                        self.z_dim_gauss, self.n_filter, self.nc, self.n_rep,
                        self.sbd, self.kernel_size, self.padding, self.AE)

        print("CUDA availability: " + str(torch.cuda.is_available()))
        self.device = torch.device(
            "cuda:0" if torch.cuda.is_available() else "cpu")
        if torch.cuda.device_count() > 1:
            print("Let's use", torch.cuda.device_count(), "GPUs!")
            net = nn.DataParallel(net)

        self.net = net.to(self.device)
        print("net on cuda: " + str(next(self.net.parameters()).is_cuda))

        self.lr = args.lr
        self.beta1 = args.beta1
        self.beta2 = args.beta2
        if not self.freeze_decoder:
            if args.optim_type == 'Adam':
                self.optim = optim.Adam(self.net.parameters(),
                                        lr=self.lr,
                                        betas=(self.beta1, self.beta2))
            elif args.optim_type == 'SGD':
                self.optim = optim.SGD(self.net.parameters(),
                                       lr=self.lr,
                                       momentum=0.9)
        elif self.freeze_decoder:
            if args.optim_type == 'Adam':
                self.optim = optim.Adam(filter(lambda p: p.requires_grad,
                                               self.net.parameters()),
                                        lr=self.lr,
                                        betas=(self.beta1, self.beta2))
            elif args.optim_type == 'SGD':
                self.optim = optim.SGD(filter(lambda p: p.requires_grad,
                                              self.net.parameters()),
                                       lr=self.lr,
                                       momentum=0.9)

        self.save_output = args.save_output
        self.output_dir = args.output_dir  #os.path.join(args.output_dir, args.viz_name)
        print(self.output_dir)
        if not os.path.exists(self.output_dir):
            os.makedirs(self.output_dir, exist_ok=True)
        self.ckpt_dir = os.path.join(args.output_dir, args.viz_name)
        if not os.path.exists(self.ckpt_dir):
            os.makedirs(self.ckpt_dir, exist_ok=True)
        self.ckpt_name = args.ckpt_name
        if self.ckpt_name is not None:
            self.load_checkpoint(self.ckpt_name)

        if self.freeze_decoder:
            print("FREEZING DECODER")
            child_counter = 0
            for child in self.net.children():
                if child_counter == 1:
                    print("In decoder: ")
                    not_freeze = [
                        'Lin_1.weight', 'Lin_1.bias', 'Lin_2.weight',
                        'Lin_2.bias', 'Lin_3.weight', 'Lin_3.bias'
                    ]
                    for name, param in child.named_parameters():
                        if name not in not_freeze:
                            param.requires_grad = False
                            print(name, 'is FROZEN')
                        else:
                            print(name, 'is NOT frozen')
                            if name == 'Lin_1.weight' or name == 'Lin_2.weight' or name == 'Lin_3.weight':
                                init.kaiming_normal(param, nonlinearity='relu')
                            else:
                                param.data.fill_(0)

                child_counter += 1

        #print parameters in model
        encoder_size = 0
        decoder_size = 0
        for name, param in net.named_parameters():
            if param.requires_grad:
                if 'encoder' in name:
                    encoder_size += param.numel()
                elif 'decoder' in name:
                    decoder_size += param.numel()
        tot_size = encoder_size + decoder_size
        print(tot_size, "parameters in the network!", encoder_size,
              " in the encoder", decoder_size, "in the decoder")
        self.params = tot_size

        self.train_dl, self.gnrl_dl, self.gnrl_data = return_data_unsupervised(
            args)

        self.max_epoch = args.max_epoch
        self.global_iter = 0
        self.gather_step = args.gather_step
        self.display_step = args.display_step
        self.save_step = args.save_step

        self.beta = args.beta
        self.gamma = args.gamma
        self.l2_loss = args.l2_loss
        self.encoder_target_type = args.encoder_target_type
        self.image_size = args.image_size

        self.flip = args.flip
        if self.flip == True:
            self.flip_idx = pickle.load(
                open("{}train_idx_to_flip.p".format(args.dset_dir), "rb"))
            self.flip_idx.sort()
            print(self.flip_idx[0:20])
            print(len(self.flip_idx), " flipped images!")

        self.testing_method = args.testing_method
        self.gather = DataGather(self.testing_method, self.encoder_target_type,
                                 self.n_digits)

        self.viz_name = args.viz_name
        self.viz_port = args.viz_port
        self.viz_on = args.viz_on
        self.win_recon = None
        self.win_kld = None
        self.win_mu = None
        self.win_var = None
Exemplo n.º 35
0
def _weights_init(m):
    classname = m.__class__.__name__
    if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):
        init.kaiming_normal(m.weight)
    def __init__(self,
                 block,
                 layers,
                 network_type,
                 num_classes,
                 att_type=None):
        self.inplanes = 64
        super(ResNet, self).__init__()
        self.network_type = network_type
        # different model config between ImageNet and CIFAR
        if network_type == "ImageNet":
            self.conv1 = nn.Conv2d(3,
                                   64,
                                   kernel_size=7,
                                   stride=2,
                                   padding=3,
                                   bias=False)
            self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
            self.avgpool = nn.AvgPool2d(7)
        else:
            self.conv1 = nn.Conv2d(3,
                                   64,
                                   kernel_size=3,
                                   stride=1,
                                   padding=1,
                                   bias=False)

        self.bn1 = nn.BatchNorm2d(64)
        self.relu = nn.ReLU(inplace=True)

        if att_type == 'BAM':
            self.bam1 = BAM(64 * block.expansion)
            self.bam2 = BAM(128 * block.expansion)
            self.bam3 = BAM(256 * block.expansion)
        else:
            self.bam1, self.bam2, self.bam3 = None, None, None

        self.layer1 = self._make_layer(block, 64, layers[0], att_type=att_type)
        self.layer2 = self._make_layer(block,
                                       128,
                                       layers[1],
                                       stride=2,
                                       att_type=att_type)
        self.layer3 = self._make_layer(block,
                                       256,
                                       layers[2],
                                       stride=2,
                                       att_type=att_type)
        self.layer4 = self._make_layer(block,
                                       512,
                                       layers[3],
                                       stride=2,
                                       att_type=att_type)

        self.fc = nn.Linear(512 * block.expansion, num_classes)

        init.kaiming_normal(self.fc.weight)
        for key in self.state_dict():
            if key.split('.')[-1] == "weight":
                if "conv" in key:
                    init.kaiming_normal(self.state_dict()[key], mode='fan_out')
                if "bn" in key:
                    if "SpatialGate" in key:
                        self.state_dict()[key][...] = 0
                    else:
                        self.state_dict()[key][...] = 1
            elif key.split(".")[-1] == 'bias':
                self.state_dict()[key][...] = 0
Exemplo n.º 37
0
    def __init__(self, dim, output_padding, numpen, dropout=0.2, slope=0.0):
        super(convSDAE, self).__init__()
        self.in_dim = dim[0]
        self.nlayers = len(dim) - 1
        self.reluslope = slope
        self.numpen = numpen
        self.enc, self.dec = [], []
        self.benc, self.bdec = [], []
        for i in range(self.nlayers):
            if i == self.nlayers - 1:
                self.enc.append(nn.Linear(dim[i] * numpen * numpen,
                                          dim[i + 1]))
                self.benc.append(nn.BatchNorm2d(dim[i + 1]))
                self.dec.append(
                    nn.ConvTranspose2d(dim[i + 1],
                                       dim[i],
                                       kernel_size=numpen,
                                       stride=1))
                self.bdec.append(nn.BatchNorm2d(dim[i]))
            elif i == 0:
                self.enc.append(
                    nn.Conv2d(dim[i],
                              dim[i + 1],
                              kernel_size=4,
                              stride=2,
                              padding=1))
                self.benc.append(nn.BatchNorm2d(dim[i + 1]))
                self.dec.append(
                    nn.ConvTranspose2d(dim[i + 1],
                                       dim[i],
                                       kernel_size=4,
                                       stride=2,
                                       padding=1,
                                       output_padding=output_padding[i]))
                self.bdec.append(nn.BatchNorm2d(dim[i]))
            else:
                self.enc.append(
                    nn.Conv2d(dim[i],
                              dim[i + 1],
                              kernel_size=5,
                              stride=2,
                              padding=2))
                self.benc.append(nn.BatchNorm2d(dim[i + 1]))
                self.dec.append(
                    nn.ConvTranspose2d(dim[i + 1],
                                       dim[i],
                                       kernel_size=5,
                                       stride=2,
                                       padding=2,
                                       output_padding=output_padding[i]))
                self.bdec.append(nn.BatchNorm2d(dim[i]))
            setattr(self, 'enc_{}'.format(i), self.enc[-1])
            setattr(self, 'benc_{}'.format(i), self.benc[-1])
            setattr(self, 'dec_{}'.format(i), self.dec[-1])
            setattr(self, 'bdec_{}'.format(i), self.bdec[-1])
        self.base = []
        self.bbase = []
        for i in range(self.nlayers):
            self.base.append(nn.Sequential(*self.enc[:i]))
            self.bbase.append(nn.Sequential(*self.benc[:i]))
        self.dropmodule1 = nn.Dropout(p=dropout)
        self.dropmodule2 = nn.Dropout(p=dropout)
        self.loss = nn.MSELoss(size_average=True)

        # initialization
        for m in self.modules():
            if isinstance(m, nn.Linear):
                init.normal(m.weight, std=1e-2)
                if m.bias.data is not None:
                    init.constant(m.bias, 0)
            elif isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
                init.kaiming_normal(m.weight, mode='fan_out')
                if m.bias.data is not None:
                    init.constant(m.bias, 0)
Exemplo n.º 38
0
def conv_init(ni, no, k):
    return kaiming_normal(torch.Tensor(no, ni, k, k))
Exemplo n.º 39
0
def init_conv(conv, glu=True):
    init.kaiming_normal(conv.weight)
    if conv.bias is not None:
        conv.bias.data.zero_()
Exemplo n.º 40
0
    def __init__(self,
                 depth,
                 pretrained=True,
                 cut_at_pooling=False,
                 num_features=0,
                 norm=False,
                 dropout=0,
                 num_classes=0):
        super(ResNet, self).__init__()

        self.depth = depth
        self.pretrained = pretrained
        self.cut_at_pooling = cut_at_pooling

        # Construct base (pretrained) resnet
        if depth not in ResNet.__factory:
            raise KeyError("Unsupported depth:", depth)

        self.base = ResNet.__factory[depth](pretrained=pretrained)

        # fix layers [conv1 ~ layer2]
        fixed_names = []
        for name, module in self.base._modules.items():
            if name == "layer3":
                assert fixed_names == [
                    "conv1", "bn1", "relu", "maxpool", "layer1", "layer2"
                ]
                break
            fixed_names.append(name)
            for param in module.parameters():
                param.requires_grad = False

        if not self.cut_at_pooling:
            self.num_features = num_features
            self.norm = norm
            self.dropout = dropout
            self.has_embedding = num_features > 0
            self.num_classes = num_classes

            out_planes = self.base.fc.in_features

            # Append new layers
            if self.has_embedding:
                self.feat = nn.Linear(out_planes, self.num_features)
                self.feat_bn = nn.BatchNorm1d(self.num_features)
                init.kaiming_normal(self.feat.weight, mode='fan_out')
                init.constant(self.feat.bias, 0)
                init.constant(self.feat_bn.weight, 1)
                init.constant(self.feat_bn.bias, 0)
            else:
                # Change the num_features to CNN output channels
                self.num_features = out_planes
            if self.dropout > 0:
                self.drop = nn.Dropout(self.dropout)
            if self.num_classes > 0:
                self.classifier = nn.Linear(self.num_features,
                                            self.num_classes)
                init.normal(self.classifier.weight, std=0.001)
                init.constant(self.classifier.bias, 0)

        if not self.pretrained:
            self.reset_params()
 def __init_weights(self):
     for m in self.modules():
         if isinstance(m, nn.Conv1d):
             kaiming_normal(m.weight, mode='fan_in')
             if m.bias is not None:
                 constant(m.bias, 0)
def init_weights_he(model):
    if isinstance(model, nn.Conv2d):
        init.kaiming_normal(model.weight)
        init.constant(model.bias, 0)
Exemplo n.º 43
0
def linear_init(ni, no):
    return kaiming_normal(torch.Tensor(no, ni))
Exemplo n.º 44
0
def weight_init(m):
    if isinstance(m, nn.Conv2d):
        torch.nn.init.xavier_normal(m.weight.data)
    elif isinstance(m, nn.Linear):
        kaiming_normal(m.weight)
        m.bias.data.zero_()
Exemplo n.º 45
0
Arquivo: resnet.py Projeto: naiqili/CL
def _weights_init(m):
    classname = m.__class__.__name__
    #print(classname)
    if isinstance(m, nn.Linear) or isinstance(m, nn.Conv2d):
        init.kaiming_normal(m.weight)
Exemplo n.º 46
0
 def reset_parameters(self):
     init.kaiming_normal(self.attend[0].weight.data)
     init.constant(self.attend[0].bias.data, val=0)
Exemplo n.º 47
0
def init_params(net):
    '''Init layer parameters.'''
    for m in net.modules():
        if isinstance(m, nn.Conv2d):
            init.kaiming_normal(m.weight, mode='fan_out')
            if m.bias:
                init.constant(m.bias, 0)
        elif isinstance(m, nn.BatchNorm2d):
            init.constant(m.weight, 1)
            init.constant(m.bias, 0)
        elif isinstance(m, nn.Linear):
            init.normal(m.weight, std=1e-3)
            if m.bias:
                init.constant(m.bias, 0)


# _, term_width = os.popen('stty size', 'r').read().split()
# term_width = int(term_width)
#
# TOTAL_BAR_LENGTH = 65.
# last_time = time.time()
# begin_time = last_time
# def progress_bar(current, total, msg=None):
#     global last_time, begin_time
#     if current == 0:
#         begin_time = time.time()  # Reset for new bar.
#
#     cur_len = int(TOTAL_BAR_LENGTH*current/total)
#     rest_len = int(TOTAL_BAR_LENGTH - cur_len) - 1
#
#     sys.stdout.write(' [')
#     for i in range(cur_len):
#         sys.stdout.write('=')
#     sys.stdout.write('>')
#     for i in range(rest_len):
#         sys.stdout.write('.')
#     sys.stdout.write(']')
#
#     cur_time = time.time()
#     step_time = cur_time - last_time
#     last_time = cur_time
#     tot_time = cur_time - begin_time
#
#     L = []
#     L.append('  Step: %s' % format_time(step_time))
#     L.append(' | Tot: %s' % format_time(tot_time))
#     if msg:
#         L.append(' | ' + msg)
#
#     msg = ''.join(L)
#     sys.stdout.write(msg)
#     for i in range(term_width-int(TOTAL_BAR_LENGTH)-len(msg)-3):
#         sys.stdout.write(' ')
#
#     # Go back to the center of the bar.
#     for i in range(term_width-int(TOTAL_BAR_LENGTH/2)+2):
#         sys.stdout.write('\b')
#     sys.stdout.write(' %d/%d ' % (current+1, total))
#
#     if current < total-1:
#         sys.stdout.write('\r')
#     else:
#         sys.stdout.write('\n')
#     sys.stdout.flush()
#
# def format_time(seconds):
#     days = int(seconds / 3600/24)
#     seconds = seconds - days*3600*24
#     hours = int(seconds / 3600)
#     seconds = seconds - hours*3600
#     minutes = int(seconds / 60)
#     seconds = seconds - minutes*60
#     secondsf = int(seconds)
#     seconds = seconds - secondsf
#     millis = int(seconds*1000)
#
#     f = ''
#     i = 1
#     if days > 0:
#         f += str(days) + 'D'
#         i += 1
#     if hours > 0 and i <= 2:
#         f += str(hours) + 'h'
#         i += 1
#     if minutes > 0 and i <= 2:
#         f += str(minutes) + 'm'
#         i += 1
#     if secondsf > 0 and i <= 2:
#         f += str(secondsf) + 's'
#         i += 1
#     if millis > 0 and i <= 2:
#         f += str(millis) + 'ms'
#         i += 1
#     if f == '':
#         f = '0ms'
#     return f
Exemplo n.º 48
0
def conv_init(ni, no, k):
    return kaiming_normal(torch.Tensor(no, ni, k, k))
 def _init_weights(self):
     if cfg.MRCNN.CONV_INIT == 'GaussianFill':
         init.normal(self.upconv5.weight, std=0.001)
     elif cfg.MRCNN.CONV_INIT == 'MSRAFill':
         init.kaiming_normal(self.upconv5.weight)
     init.constant(self.upconv5.bias, 0)
    def reset_parameters(self):
        init.kaiming_normal(self.encoder1[0].weight.data)
        init.kaiming_normal(self.encoder1[2].weight.data)
        init.constant(self.encoder1[0].bias.data, val=0)
        init.constant(self.encoder1[2].bias.data, val=0)

        init.kaiming_normal(self.encoder2[0].weight.data)
        init.kaiming_normal(self.encoder2[2].weight.data)
        init.constant(self.encoder2[0].bias.data, val=0)
        init.constant(self.encoder2[2].bias.data, val=0)

        init.kaiming_normal(self.encoder3[0].weight.data)
        init.kaiming_normal(self.encoder3[2].weight.data)
        init.constant(self.encoder3[0].bias.data, val=0)
        init.constant(self.encoder3[2].bias.data, val=0)

        init.kaiming_normal(self.encoder4[0].weight.data)
        init.constant(self.encoder4[0].bias.data, val=0)

        init.kaiming_normal(self.decoder1[0].weight.data)
        init.kaiming_normal(self.decoder1[2].weight.data)
        init.constant(self.decoder1[0].bias.data, val=0)
        init.constant(self.decoder1[2].bias.data, val=0)

        init.kaiming_normal(self.decoder2[0].weight.data)
        init.kaiming_normal(self.decoder2[2].weight.data)
        init.constant(self.decoder2[0].bias.data, val=0)
        init.constant(self.decoder2[2].bias.data, val=0)

        init.kaiming_normal(self.decoder3[0].weight.data)
        init.kaiming_normal(self.decoder3[2].weight.data)
        init.constant(self.decoder3[0].bias.data, val=0)
        init.constant(self.decoder3[2].bias.data, val=0)

        init.kaiming_normal(self.decoder4[0].weight.data)
        init.constant(self.decoder4[0].bias.data, val=0)
Exemplo n.º 51
0
def weights_init(net):
    for m in net.modules():
        if isinstance(m, torch.nn.Conv3d) or isinstance(
                m, torch.nn.ConvTranspose3d):
            init.kaiming_normal(m.weight)
            init.constant(m.bias, 0.01)