Пример #1
0
def init_param(model):
    for name, param in model.named_parameters():
        # skip over the embeddings so that the padding index ones are 0
        if 'embed' in name:
            continue
        elif ('rnn' in name or 'lm' in name) and len(param.size()) >= 2:
            init.orthogonal(param)
        else:
            init.normal(param, 0, 0.01)
Пример #2
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    print(classname)
    if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d)):
        init.orthogonal(m.weight.data, gain=1)
    elif isinstance(m, nn.Linear):
        init.orthogonal(m.weight.data, gain=1)
    elif isinstance(m, nn.BatchNorm2d):
        init.normal(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)
 def wt_init(self):
     for name, param in self.named_parameters():
         if 'gru' in name and 'weight' in name:
             init.orthogonal(param)
         elif 'linear' in name:
             init.normal(
                 param, 0,
                 math.sqrt(2. /
                           float(self.hidden_size_1 + self.hidden_size_2 +
                                 self.hidden_size_3)))
Пример #4
0
    def __init__(self, input_size , hidden_size, n_layers=1):
        super(Encoder, self).__init__()
        self.n_layers = n_layers
        self.hidden_size = hidden_size

        #self.embedding = nn.Embedding(vocab_size, emb_size)
        self.gru = nn.GRU(input_size, hidden_size, dropout=0.2, batch_first=True, bidirectional=True)
        for w in self.gru.parameters(): # initialize the gate weights with orthogonal
            if w.dim()>1:
                weight_init.orthogonal(w)
Пример #5
0
def initWeights(net, scheme='orthogonal'):
   print('Initializing weights. Warning: may overwrite sensitive bias parameters (e.g. batchnorm)')
   for e in net.parameters():
      if scheme == 'orthogonal':
         if len(e.size()) >= 2:
            init.orthogonal(e)
      elif scheme == 'normal':
         init.normal(e, std=1e-2)
      elif scheme == 'xavier':
         init.xavier_normal(e)
Пример #6
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    #print(classname)
    if 'Conv' in classname:
        init.orthogonal(m.weight.data, gain=1)
    elif 'Linear' in classname:
        init.orthogonal(m.weight.data, gain=1)
    elif 'BatchNorm' in classname:
        init.normal(m.weight.data, 1.0, 0.02)
        init.constant(m.bias.data, 0.0)
Пример #7
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    # print(classname)
    if classname.find("Conv") != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find("Linear") != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find("BatchNorm2d") != -1:
        init.uniform(m.weight.data, 1.0, 0.02)
        init.constant(m.bias.data, 0.0)
Пример #8
0
 def reset_parameters(self):
     self.reducer.reset_parameters()
     init.kaiming_normal(self.word_linear.weight.data)
     init.constant(self.word_linear.bias.data, val=0)
     init.kaiming_normal(self.tracker_cell.weight_ih.data)
     init.orthogonal(self.tracker_cell.weight_hh.data)
     init.constant(self.tracker_cell.bias_ih.data, val=0)
     init.constant(self.tracker_cell.bias_hh.data, val=0)
     init.kaiming_normal(self.trans_linear.weight.data)
     init.constant(self.trans_linear.bias.data, val=0)
Пример #9
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    print(classname)
    if classname.find('Conv') != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find('Linear') != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find('BatchNorm2d') != -1:
        init.normal(m.weight.data, 1.0, 0.02)
        init.constant(m.bias.data, 0.0)
Пример #10
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    print(classname)
    if classname.find('Conv') != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find('Linear') != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find('BatchNorm2d') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)
Пример #11
0
    def __init__(self,
                 growth_rate=32,
                 block_config=(6, 12, 24, 16),
                 num_init_features=64,
                 bn_size=4,
                 drop_rate=0,
                 num_classes=p_transform["n_labels"]):

        super(MyDenseNet, self).__init__()

        # First convolution
        self.features = nn.Sequential(
            OrderedDict([
                ('conv0',
                 nn.Conv2d(3,
                           num_init_features,
                           kernel_size=7,
                           stride=2,
                           padding=3,
                           bias=False)),
                ('norm0', nn.BatchNorm2d(num_init_features)),
                ('relu0', nn.ReLU(inplace=True)),
                ('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
            ]))

        # Each denseblock
        num_features = num_init_features

        final_num_features = 0
        for i, num_layers in enumerate(block_config):
            block = torchvision.models.densenet._DenseBlock(
                num_layers=num_layers,
                num_input_features=num_features,
                bn_size=bn_size,
                growth_rate=growth_rate,
                drop_rate=drop_rate)
            self.features.add_module('denseblock%d' % (i + 1), block)

            num_features = num_features + num_layers * growth_rate
            if i != len(block_config) - 1:
                trans = torchvision.models.densenet._Transition(
                    num_input_features=num_features,
                    num_output_features=num_features // 2)
                self.features.add_module('transition%d' % (i + 1), trans)
                num_features = num_features // 2

        # Final batch norm
        self.features.add_module('norm5', nn.BatchNorm2d(num_features))
        #self.classifier_drop = nn.Dropout(p=0.75)
        # Linear layer
        self.fc = nn.Linear(num_features, num_features / 4)
        init.orthogonal(self.fc.weight, gain=np.sqrt(2.0))

        self.classifier = nn.Linear(num_features / 4, num_classes)
        self.classifier.weight.data.zero_()
Пример #12
0
    def _weight_init(self, m):

        if isinstance(m, nn.LSTM):

            for n, p in m.named_parameters():

                if 'weight' in n:
                    # init.xavier_normal(p.data)
                    init.orthogonal(p.data)
                elif 'bias' in n:
                    p.data.zero_()
Пример #13
0
 def reset_parameters(self):
     """
     Initialize parameters following the way proposed in the paper.
     """
     init.orthogonal(self.weight_ih.data)
     weight_hh_data = torch.eye(self.hidden_size)
     weight_hh_data = weight_hh_data.repeat(1, 3)
     self.weight_hh.data.set_(weight_hh_data)
     # The bias is just set to zero vectors.
     if self.use_bias:
         init.constant(self.bias.data, val=0)
Пример #14
0
 def init_weights(self,init_type,init_scale):
     # Initialize weight matrix
     for p in self.parameters():
         if init_type=="orthogonal" and p.dim()>=2:
             nninit.orthogonal(p)
         elif init_type=="uniform":
             p.data.uniform_(-init_scale, init_scale)
         elif init_type=="xavier_n" and p.dim()>=2:
             nninit.xavier_normal(p)
         elif init_type=="xavier_u" and p.dim()>=2:
             nninit.xavier_uniform(p)
Пример #15
0
 def __init__(self, x_dim, h_dim, act_func, W=None):
     super(AutoEncoder, self).__init__()
     self.x_dim = x_dim
     self.h_dim = h_dim
     self.f = act_func
     if W is None:
         self.W = Parameter(torch.FloatTensor(x_dim, h_dim))
         # init.xavier_uniform(self.W)
         init.orthogonal(self.W)
     else:
         self.W = Parameter(torch.FloatTensor(W))
Пример #16
0
    def __init__(self, depth, pretrained=True, cut_at_pooling=False,
                 num_features=0, norm=False, dropout=0, num_classes=0,
                 num_diff_features=0, iden_pretrain = False,
                 model_path='/media/hh/disc_d/hh/open-reid-master/pretrained model/resnet50.pth'):
        super(ResNet, self).__init__()

        self.depth = depth
        self.pretrained = pretrained
        self.cut_at_pooling = cut_at_pooling
        self.iden_pretrain = iden_pretrain

        # Construct base (pretrained) resnet
        if depth not in ResNet.__factory:
            raise KeyError("Unsupported depth:", depth)
        # self.base = ResNet.__factory[depth](pretrained=pretrained)
        self.base = baseresnet.ResNet(baseresnet.Bottleneck, [3, 4, 6, 3])
        if pretrained is True:
            self.base.load_state_dict(torch.load(model_path))
        self.relu = nn.ReLU(inplace=True)
        if not self.cut_at_pooling:
            self.num_features = num_features
            self.num_diff_features = num_diff_features
            self.norm = norm
            self.dropout = dropout
            self.has_embedding = num_features > 0
            self.num_classes = num_classes

            out_planes = self.base.fc.in_features

            # Append new layers
            if self.has_embedding:
                self.feat = nn.Linear(out_planes, self.num_features)
                self.feat_bn = nn.BatchNorm1d(self.num_features)
                init.kaiming_normal(self.feat.weight, mode='fan_out')
                init.constant(self.feat.bias, 0)
                init.constant(self.feat_bn.weight, 1)
                init.constant(self.feat_bn.bias, 0)
            else:
                # Change the num_features to CNN output channels
                self.num_features = out_planes
            if self.dropout > 0:
                self.drop = nn.Dropout(self.dropout)
            if self.num_diff_features > 0:
                self.diff_feat = nn.Linear(self.num_features, self.num_diff_features)
                init.orthogonal(self.diff_feat.weight)
                init.constant(self.diff_feat.bias, 0)
            if self.num_classes > 0:
                self.classifier = nn.Linear(self.num_features, self.num_classes)
                # init.orthogonal(self.classifier.weight)
                init.normal(self.classifier.weight, std=0.001)
                init.constant(self.classifier.bias, 0)

        if not self.pretrained:
            self.reset_params()
Пример #17
0
    def reset_parameters(self):

        init.orthogonal(self.alpha_weight_ih.data)

        alpha_weight_hh_data = torch.eye(self.hidden_size)
        alpha_weight_hh_data = alpha_weight_hh_data.repeat(1, 1)
        self.alpha_weight_hh.data.set_(alpha_weight_hh_data)


        if self.use_bias:
            init.constant(self.alpha_bias.data, val=0)
Пример #18
0
    def init_weights(self):
        init.orthogonal(self.lstm.weight_ih_l0)
        init.uniform(self.lstm.weight_hh_l0, a=-0.01, b=0.01)

        embedding_weights = torch.FloatTensor(self.vocab_size, 100)
        init.uniform(embedding_weights, a=-0.25, b=0.25)

        for id, vec in id_to_vec.items():
            embedding_weights[id] = vec

        self.embedding.weight.data.copy_(embedding_weights)
Пример #19
0
    def init_bilstm(self, hidden_dim):
        init.xavier_uniform(self.encoder.weight_ih_l0)
        init.xavier_uniform(self.encoder.weight_ih_l0_reverse)
        init.orthogonal(self.encoder.weight_hh_l0)
        init.orthogonal(self.encoder.weight_hh_l0_reverse)

        bias = self.init_lstm_bias(self.encoder.bias_ih_l0, hidden_dim)
        self.encoder.bias_ih_l0 = nn.Parameter(bias.clone())
        self.encoder.bias_hh_l0 = nn.Parameter(bias.clone())
        self.encoder.bias_ih_l0_reverse = nn.Parameter(bias.clone())
        self.encoder.bias_hh_l0_reverse = nn.Parameter(bias.clone())
Пример #20
0
    def init_weights(self):
        init.uniform(self.lstm.weight_ih_l0, a=-0.01, b=0.01)
        init.orthogonal(self.lstm.weight_hh_l0)
        self.lstm.weight_ih_l0.requires_grad = True
        self.lstm.weight_hh_l0.requires_grad = True

        embedding_weights = torch.FloatTensor(self.vocab_size, self.input_size)

        for id, vec in id_to_vec.items():
            embedding_weights[id] = vec

        self.embedding.weight = nn.Parameter(embedding_weights, requires_grad=True)
Пример #21
0
  def init_weights(self):
    init.orthogonal(self.rnn.weight_ih_l0)
    init.uniform(self.rnn.weight_hh_l0, a=-0.01, b=0.01)

    glove_embeddings = preprocessing.load_glove_embeddings()
    embedding_weights = torch.FloatTensor(self.vocab_size, self.input_size)
    init.uniform(embedding_weights, a=-0.25, b=0.25)
    for k,v in glove_embeddings.items():
      embedding_weights[k] = torch.FloatTensor(v)
    embedding_weights[0] = torch.FloatTensor([0]*self.input_size)
    del self.embedding.weight
    self.embedding.weight = nn.Parameter(embedding_weights)
Пример #22
0
    def init_weights(self):
        init_range = self._init_range
        init_std = self._gru_init_std

        self._l1_embedding_layer.weight.data.copy_(
            torch.from_numpy(self._l1_emb_vector))
        self._l2_embedding_layer.weight.data.copy_(
            torch.from_numpy(self._l2_emb_vector))

        unk_n_var_1 = self._l1_embedding_layer.weight.data[1:2 + self._nr_unk +
                                                           self._var_size, :]
        init.normal(unk_n_var_1, 0, 1)
        unk_n_var_1 /= torch.norm(unk_n_var_1, p=2, dim=1).unsqueeze(
            1)  # normalise randomly initialised embeddings
        self._l1_embedding_layer.weight.data[0, :] = 0
        if not self._emb_trainable:
            self._l1_embedding_layer.weight.requires_gard = False  # size = entities + ph + non-ent-marker

        unk_2 = self._l2_embedding_layer.weight.data[2:2 + self._nr_unk, :]
        init.normal(unk_2, 0, 1)
        unk_2 /= torch.norm(unk_2, p=2, dim=1).unsqueeze(
            1)  # normalise randomly initialised embeddings
        # ^^^ init unk * 100 embeddings
        self._l2_embedding_layer.weight.data[
            1, :] = self._l1_embedding_layer.weight.data[1, :]
        # ^^^ share @placeholder embedding
        self._l2_embedding_layer.weight.data[2 + self._nr_unk: 2 + self._nr_unk + self._var_size, :] = \
            self._l1_embedding_layer.weight.data[2 + self._nr_unk: 2 + self._nr_unk + self._var_size, :]
        # ^^^ share @entityX embeddings
        self._l2_embedding_layer.weight.data[0, :] = 0
        if not self._emb_trainable:
            self._l2_embedding_layer.weight.requires_gard = False  # size = entities + ph + non-ent-marker
        # DONE: initialise non-zero locations
        # TODO: randomise in forward step?

        gain = init.calculate_gain('tanh')
        for p in self._recurrent_layer.parameters():
            if p.dim() == 1:
                p.data.normal_(0, init_std)
            else:
                init.orthogonal(p.data, gain)
        for p in self._question_recurrent_layer.parameters():
            if p.dim() == 1:
                p.data.normal_(0, init_std)
            else:
                init.orthogonal(p.data, gain)

        # self._embedding_projection_layer.weight.data.uniform_(-init_range, init_range)
        self._output_layer.weight.data.uniform_(-init_range, init_range)
        self._output_layer.bias.data.fill_(0)

        self._mix_matrix.data.uniform_(-init_range, init_range)
Пример #23
0
 def _initialize_weights(self):
     """" ""
     if type == 'Tanh':
         self.nonlinearity = 'tanh'
     elif type == 'ReLU':
         self.nonlinearity = 'relu'
     elif type == 'Leaky':
         self.nonlinearity = 'leaky_relu'
         """
     init.orthogonal(self.conv1.weight,
                     init.calculate_gain(self.nonlinearity))
     init.orthogonal(self.conv2.weight,
                     init.calculate_gain(self.nonlinearity))
Пример #24
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        init.orthogonal(m.weight.data, gain=1)
        if m.bias is not None:
            m.bias.data.zero_()
    elif classname.find('Linear') != -1:
        init.orthogonal(m.weight.data, gain=1)
        if m.bias is not None:
            m.bias.data.zero_()
    elif classname.find('BatchNorm2d') != -1:
        init.constant_(m.weight.data, 1.0)
        init.constant_(m.bias.data, 0.0)
Пример #25
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    # print(classname)
    if classname.find('Conv') != -1:
        try:
            init.orthogonal(m.weight.data, gain=1)
        except AttributeError:
            weights_init_normal(m)
    elif classname.find('Linear') != -1:
        init.orthogonal(m.weight.data, gain=1)
    elif classname.find('BatchNorm2d') != -1:
        init.normal(m.weight.data, 1.0, 0.02)
        init.constant(m.bias.data, 0.0)
Пример #26
0
 def init_weights(self, init_type, init_scale):
     # Initialize weight matrix
     for p in self.parameters():
         if init_type == "orthogonal" and p.dim() >= 2:
             nninit.orthogonal(p)
         elif init_type == "uniform":
             p.data.uniform_(-init_scale, init_scale)
         elif init_type == "xavier_n" and p.dim() >= 2:
             nninit.xavier_normal(p)
         elif init_type == "xavier_u" and p.dim() >= 2:
             nninit.xavier_uniform(p)
     # Initialize bias for the linear layer
     self.hidden2tag.bias.data.fill_(0.0)
Пример #27
0
def torch_weight_init(m):
    """
    Usage:
        model = Model()
        model.apply(weight_init)
    """
    if isinstance(m, nn.Conv1d):
        init.normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.Conv2d):
        init.xavier_normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.Conv3d):
        init.xavier_normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.ConvTranspose1d):
        init.normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.ConvTranspose2d):
        init.xavier_normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.ConvTranspose3d):
        init.xavier_normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.BatchNorm1d):
        init.normal(m.weight.data, mean=1, std=0.02)
        init.constant(m.bias.data, 0)
    elif isinstance(m, nn.BatchNorm2d):
        init.normal(m.weight.data, mean=1, std=0.02)
        init.constant(m.bias.data, 0)
    elif isinstance(m, nn.BatchNorm3d):
        init.normal(m.weight.data, mean=1, std=0.02)
        init.constant(m.bias.data, 0)
    elif isinstance(m, nn.Linear):
        init.xavier_normal(m.weight.data)
        init.normal(m.bias.data)
    elif isinstance(m, nn.LSTM):
        for param in m.parameters():
            if len(param.shape) >= 2:
                init.orthogonal(param.data)
            else:
                init.normal(param.data)
    elif isinstance(m, nn.LSTMCell):
        for param in m.parameters():
            if len(param.shape) >= 2:
                init.orthogonal(param.data)
            else:
                init.normal(param.data)
    elif isinstance(m, nn.GRU):
        for param in m.parameters():
            if len(param.shape) >= 2:
                init.orthogonal(param.data)
            else:
                init.normal(param.data)
    elif isinstance(m, nn.GRUCell):
        for param in m.parameters():
            if len(param.shape) >= 2:
                init.orthogonal(param.data)
            else:
                init.normal(param.data)
Пример #28
0
    def reset_parameters(self):
        """
        Initialize parameters following the way proposed in the paper.
        """

        # The input-to-hidden weight matrix is initialized orthogonally.
        init.orthogonal(self.weight_ih.data)
        # The hidden-to-hidden weight matrix is initialized as an identity
        # matrix.
        weight_hh_data = torch.eye(self.hidden_size)
        weight_hh_data = weight_hh_data.repeat(1, 3)
        self.weight_hh.data.set_(weight_hh_data)
        # The bias is just set to zero vectors.
        init.constant(self.bias.data, val=0)
Пример #29
0
    def __init__(self, input_size, hidden_size, vocab_size, n_layers=1):
        super(Decoder, self).__init__()
        self.n_layers = n_layers
        self.hidden_size = hidden_size
        self.vocab_size=vocab_size

        #self.embedding = nn.Embedding(vocab_size, emb_size)
        self.gru = nn.GRU(input_size, hidden_size, dropout=0.2, batch_first=True)
        for w in self.gru.parameters(): # initialize the gate weights with orthogonal
            if w.dim()>1:
                weight_init.orthogonal(w)
        self.out = nn.Linear(hidden_size, vocab_size)
        self.softmax = nn.LogSoftmax()
        self.dropout=nn.Dropout(p=0.3)
Пример #30
0
    def __init__(self, in_dim, out_dim, p_dropout=0.0):
        super().__init__()

        self.input_dim = in_dim
        self.output_dim = out_dim
        self.dropout = p_dropout

        self.W_i = nn.Linear(in_dim, out_dim)
        init.xavier_uniform(self.W_i.weight)
        self.W_i.bias = nn.Parameter(torch.FloatTensor(out_dim).zero_())
        self.U_i = nn.Linear(out_dim, out_dim, bias=False)
        init.orthogonal(self.U_i.weight)

        self.W_f = nn.Linear(in_dim, out_dim)
        init.xavier_uniform(self.W_f.weight)
        self.W_f.bias = nn.Parameter(torch.FloatTensor(out_dim).fill_(1.0))
        self.U_f = nn.Linear(out_dim, out_dim)
        init.orthogonal(self.U_f.weight)

        self.W_c = nn.Linear(in_dim, out_dim)
        init.xavier_uniform(self.W_c.weight)
        self.W_c.bias = nn.Parameter(torch.FloatTensor(out_dim).fill_(0.0))
        self.U_c = nn.Linear(out_dim, out_dim)
        init.orthogonal(self.U_c.weight)

        self.W_o = nn.Linear(in_dim, out_dim)
        init.xavier_uniform(self.W_o.weight)
        self.W_o.bias = nn.Parameter(torch.FloatTensor(out_dim).fill_(0.0))
        self.U_o = nn.Linear(out_dim, out_dim)
        init.orthogonal(self.U_o.weight)
Пример #31
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    #    print(classname)
    if classname.find('Conv2d') != -1 or classname.find('ConvTranspose') != -1:
        init.orthogonal(m.weight.data, gain=1)
        if m.bias is not None:
            m.bias.data.zero_()
    elif classname.find('Linear') != -1:
        init.orthogonal(m.weight.data, gain=1)
        if m.bias is not None:
            m.bias.data.zero_()
    elif classname.find('BatchNorm2d') != -1:
        init.normal_(m.weight.data, 1.0, 0.02)
        init.constant_(m.bias.data, 0.0)
Пример #32
0
    def __init__(self, num_features=0, norm=False, dropout=0,
                 num_diff_features=0):
        super(Trip_embedding, self).__init__()

        self.num_features = num_features
        self.num_diff_features = num_diff_features
        self.norm = norm
        self.dropout = dropout
        if self.dropout > 0:
            self.drop = nn.Dropout(self.dropout)
        if self.num_diff_features > 0:
            self.diff_feat = nn.Linear(self.num_features, self.num_diff_features)
            init.orthogonal(self.diff_feat.weight)
            init.constant(self.diff_feat.bias, 0)
Пример #33
0
    def initialize_encoder(self):
        """Manual weight/bias initialization.
        """
        xavier_normal(self.gru_enc_f.weight_ih)
        orthogonal(self.gru_enc_f.weight_hh)

        self.gru_enc_f.bias_ih.data.zero_()
        self.gru_enc_f.bias_hh.data.zero_()

        xavier_normal(self.gru_enc_b.weight_ih)
        orthogonal(self.gru_enc_b.weight_hh)

        self.gru_enc_b.bias_ih.data.zero_()
        self.gru_enc_b.bias_hh.data.zero_()
Пример #34
0
def weights_init_orthogonal(m):
    classname = m.__class__.__name__
    # print('initializing [%s] ...' % classname)
    if classname.find('Conv') != -1:
        init.orthogonal(m.weight.data, gain=1)
        if m.bias is not None:
            m.bias.data.zero_()
    elif classname.find('Linear') != -1:
        init.orthogonal(m.weight.data, gain=1)
        if m.bias is not None:
            m.bias.data.zero_()
    elif classname.find('BatchNorm2d') != -1:
        init.normal(m.weight.data, 1.0, 0.02)
        init.constant(m.bias.data, 0.0)
 def init_func(m):
     classname = m.__class__.__name__
     if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1):
         if init_type == 'normal':
             init.normal(m.weight.data, 0.0, gain)
         elif init_type == 'xavier':
             init.xavier_normal(m.weight.data, gain=gain)
         elif init_type == 'kaiming':
             init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
         elif init_type == 'orthogonal':
             init.orthogonal(m.weight.data, gain=gain)
         else:
             raise NotImplementedError('initialization method [%s] is not implemented' % init_type)
         if hasattr(m, 'bias') and m.bias is not None:
             init.constant(m.bias.data, 0.0)
     elif classname.find('BatchNorm2d') != -1:
         init.normal(m.weight.data, 1.0, gain)
         init.constant(m.bias.data, 0.0)
Пример #36
0
 def init_fun(m):
     classname = m.__class__.__name__
     if (classname.find('Conv') == 0 or classname.find('Linear') == 0) and hasattr(m, 'weight'):
         # print m.__class__.__name__
         if init_type == 'gaussian':
             init.normal(m.weight.data, 0.0, 0.02)
         elif init_type == 'xavier':
             init.xavier_normal(m.weight.data, gain=math.sqrt(2))
         elif init_type == 'kaiming':
             init.kaiming_normal(m.weight.data, a=0, mode='fan_in')
         elif init_type == 'orthogonal':
             init.orthogonal(m.weight.data, gain=math.sqrt(2))
         elif init_type == 'default':
             pass
         else:
             assert 0, "Unsupported initialization: {}".format(init_type)
         if hasattr(m, 'bias') and m.bias is not None:
             init.constant(m.bias.data, 0.0)
Пример #37
0
 def __init__(self, margin=0, num_feature=128):
     super(AdaptTripletLoss, self).__init__()
     self.margin = margin
     self.ranking_loss = nn.MarginRankingLoss(margin=margin)
     self.softmargin_loss = nn.SoftMarginLoss()
     self.num_classes = num_feature
     self.adp_metric_embedding1 = nn.Linear(3*self.num_classes, 3*self.num_classes, bias=False)
     self.adp_metric_embedding1_bn = nn.BatchNorm1d(3*self.num_classes)
     self.adp_metric_embedding2 = nn.Linear(3*self.num_classes, 2*self.num_classes, bias=False)
     self.adp_metric_embedding2_bn = nn.BatchNorm1d(2 * self.num_classes)
     self.adp_metric_embedding3 = nn.Linear(2*self.num_classes, 2*self.num_classes, bias=False)
     self.adp_metric_embedding3_bn = nn.BatchNorm1d(2 * self.num_classes)
     self.adp_metric_embedding4 = nn.Linear(2*self.num_classes, 2*self.num_classes, bias=False)
     # self.adp_metric_embedding2_bn = nn.BatchNorm1d(self.num_classes)
     # init.constant(self.adp_metric_embedding1.bias,0)
     # init.constant(self.adp_metric_embedding2.bias,0)
     init.kaiming_normal(self.adp_metric_embedding1.weight, mode='fan_out')
     init.kaiming_normal(self.adp_metric_embedding2.weight, mode='fan_out')
     init.kaiming_normal(self.adp_metric_embedding3.weight, mode='fan_out')
     init.orthogonal(self.adp_metric_embedding4.weight)
Пример #38
0
    def reset_parameters(self):
        """
        Initialize parameters following the way proposed in the paper.
        """

        # The input-to-hidden weight matrix is initialized orthogonally.
        init.orthogonal(self.weight_ih.data)
        # The hidden-to-hidden weight matrix is initialized as an identity
        # matrix.
        weight_hh_data = torch.eye(self.hidden_size)
        weight_hh_data = weight_hh_data.repeat(4, 1)
        self.weight_hh.data.set_(weight_hh_data)
        # The bias is just set to zero vectors.
        init.constant(self.bias.data, val=0)
        # Initialization of BN parameters.
        self.bn_ih.reset_parameters()
        self.bn_hh.reset_parameters()
        self.bn_c.reset_parameters()
        self.bn_ih.bias.data.fill_(0)
        self.bn_hh.bias.data.fill_(0)
        self.bn_ih.weight.data.fill_(0.1)
        self.bn_hh.weight.data.fill_(0.1)
        self.bn_c.weight.data.fill_(0.1)
 def _initialize_weights(self):
     init.orthogonal(self.conv1.weight, init.calculate_gain('relu'))
     init.orthogonal(self.conv2.weight, init.calculate_gain('relu'))
     init.orthogonal(self.conv3.weight, init.calculate_gain('relu'))
     init.orthogonal(self.conv4.weight)
Пример #40
0
def _initialize_orthogonal(conv):
    prelu_gain = math.sqrt(2)
    init.orthogonal(conv.weight, gain=prelu_gain)
    if conv.bias is not None:
        conv.bias.data.zero_()