def __init__( self, last_conv_stride=1, last_conv_dilation=1, num_stripes=6, local_conv_out_channels=256, num_classes=0 ): super(PCBModel, self).__init__() self.base = resnet50( pretrained=True, last_conv_stride=last_conv_stride, last_conv_dilation=last_conv_dilation) self.num_stripes = num_stripes self.local_conv_list = nn.ModuleList() for _ in range(num_stripes): self.local_conv_list.append(nn.Sequential( nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True) )) if num_classes > 0: self.fc_list = nn.ModuleList() for _ in range(num_stripes): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal(fc.weight, std=0.001) init.constant(fc.bias, 0) self.fc_list.append(fc)
def init_param(model): for name, param in model.named_parameters(): # skip over the embeddings so that the padding index ones are 0 if 'embed' in name: continue elif ('rnn' in name or 'lm' in name) and len(param.size()) >= 2: init.orthogonal(param) else: init.normal(param, 0, 0.01)
def weights_init_kaiming(m): classname = m.__class__.__name__ # print(classname) if classname.find('Conv') != -1: init.kaiming_normal(m.weight.data, a=0, mode='fan_in') elif classname.find('Linear') != -1: init.kaiming_normal(m.weight.data, a=0, mode='fan_in') elif classname.find('BatchNorm2d') != -1: init.normal(m.weight.data, 1.0, 0.02) init.constant(m.bias.data, 0.0)
def weights_init_orthogonal(m): classname = m.__class__.__name__ print(classname) if classname.find('Conv') != -1: init.orthogonal(m.weight.data, gain=1) elif classname.find('Linear') != -1: init.orthogonal(m.weight.data, gain=1) elif classname.find('BatchNorm2d') != -1: init.normal(m.weight.data, 1.0, 0.02) init.constant(m.bias.data, 0.0)
def __init__(self, version=1.0, num_classes=1000): super(SqueezeNet, self).__init__() if version not in [1.0, 1.1]: raise ValueError("Unsupported SqueezeNet version {version}:" "1.0 or 1.1 expected".format(version=version)) self.num_classes = num_classes if version == 1.0: self.features = nn.Sequential( nn.Conv2d(3, 96, kernel_size=7, stride=2), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True), Fire(96, 16, 64, 64), Fire(128, 16, 64, 64), Fire(128, 32, 128, 128), nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True), Fire(256, 32, 128, 128), Fire(256, 48, 192, 192), Fire(384, 48, 192, 192), Fire(384, 64, 256, 256), nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True), Fire(512, 64, 256, 256), ) else: self.features = nn.Sequential( nn.Conv2d(3, 64, kernel_size=3, stride=2), nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True), Fire(64, 16, 64, 64), Fire(128, 16, 64, 64), nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True), Fire(128, 32, 128, 128), Fire(256, 32, 128, 128), nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True), Fire(256, 48, 192, 192), Fire(384, 48, 192, 192), Fire(384, 64, 256, 256), Fire(512, 64, 256, 256), ) # Final convolution is initialized differently form the rest final_conv = nn.Conv2d(512, self.num_classes, kernel_size=1) self.classifier = nn.Sequential( nn.Dropout(p=0.5), final_conv, nn.ReLU(inplace=True), nn.AvgPool2d(13, stride=1) ) for m in self.modules(): if isinstance(m, nn.Conv2d): if m is final_conv: init.normal(m.weight.data, mean=0.0, std=0.01) else: init.kaiming_uniform(m.weight.data) if m.bias is not None: m.bias.data.zero_()
def __init__(self, depth, pretrained=True, cut_at_pooling=False, num_features=0, norm=False, dropout=0, num_classes=0, num_diff_features=0, iden_pretrain = False, model_path='/media/hh/disc_d/hh/open-reid-master/pretrained model/resnet50.pth'): super(ResNet, self).__init__() self.depth = depth self.pretrained = pretrained self.cut_at_pooling = cut_at_pooling self.iden_pretrain = iden_pretrain # Construct base (pretrained) resnet if depth not in ResNet.__factory: raise KeyError("Unsupported depth:", depth) # self.base = ResNet.__factory[depth](pretrained=pretrained) self.base = baseresnet.ResNet(baseresnet.Bottleneck, [3, 4, 6, 3]) if pretrained is True: self.base.load_state_dict(torch.load(model_path)) self.relu = nn.ReLU(inplace=True) if not self.cut_at_pooling: self.num_features = num_features self.num_diff_features = num_diff_features self.norm = norm self.dropout = dropout self.has_embedding = num_features > 0 self.num_classes = num_classes out_planes = self.base.fc.in_features # Append new layers if self.has_embedding: self.feat = nn.Linear(out_planes, self.num_features) self.feat_bn = nn.BatchNorm1d(self.num_features) init.kaiming_normal(self.feat.weight, mode='fan_out') init.constant(self.feat.bias, 0) init.constant(self.feat_bn.weight, 1) init.constant(self.feat_bn.bias, 0) else: # Change the num_features to CNN output channels self.num_features = out_planes if self.dropout > 0: self.drop = nn.Dropout(self.dropout) if self.num_diff_features > 0: self.diff_feat = nn.Linear(self.num_features, self.num_diff_features) init.orthogonal(self.diff_feat.weight) init.constant(self.diff_feat.bias, 0) if self.num_classes > 0: self.classifier = nn.Linear(self.num_features, self.num_classes) # init.orthogonal(self.classifier.weight) init.normal(self.classifier.weight, std=0.001) init.constant(self.classifier.bias, 0) if not self.pretrained: self.reset_params()
def __init__(self, local_conv_out_channels=128, num_classes=None): super(Model, self).__init__() self.base = resnet50(pretrained=True) planes = 2048 self.local_conv = nn.Conv2d(planes, local_conv_out_channels, 1) self.local_bn = nn.BatchNorm2d(local_conv_out_channels) self.local_relu = nn.ReLU(inplace=True) if num_classes is not None: self.fc = nn.Linear(planes, num_classes) init.normal(self.fc.weight, std=0.001) init.constant(self.fc.bias, 0)
def reset_params(self): for m in self.modules(): if isinstance(m, nn.Conv2d): init.kaiming_normal(m.weight, mode='fan_out') if m.bias is not None: init.constant(m.bias, 0) elif isinstance(m, nn.BatchNorm2d): init.constant(m.weight, 1) init.constant(m.bias, 0) elif isinstance(m, nn.Linear): init.normal(m.weight, std=0.001) if m.bias is not None: init.constant(m.bias, 0)
def init_params(net): '''Init layer parameters.''' for m in net.modules(): if isinstance(m, nn.Conv2d): init.kaiming_normal(m.weight, mode='fan_out') if m.bias: init.constant(m.bias, 0) elif isinstance(m, nn.BatchNorm2d): init.constant(m.weight, 1) init.constant(m.bias, 0) elif isinstance(m, nn.Linear): init.normal(m.weight, std=1e-3) if m.bias: init.constant(m.bias, 0)
def _prepare_tsn(self, num_class): feature_dim = getattr(self.base_model, self.base_model.last_layer_name).in_features if self.dropout == 0: setattr(self.base_model, self.base_model.last_layer_name, nn.Linear(feature_dim, num_class)) self.new_fc = None else: setattr(self.base_model, self.base_model.last_layer_name, nn.Dropout(p=self.dropout)) self.new_fc = nn.Linear(feature_dim, num_class) std = 0.001 if self.new_fc is None: normal(getattr(self.base_model, self.base_model.last_layer_name).weight, 0, std) constant(getattr(self.base_model, self.base_model.last_layer_name).bias, 0) else: normal(self.new_fc.weight, 0, std) constant(self.new_fc.bias, 0) return feature_dim
def init_func(m): classname = m.__class__.__name__ if hasattr(m, 'weight') and (classname.find('Conv') != -1 or classname.find('Linear') != -1): if init_type == 'normal': init.normal(m.weight.data, 0.0, gain) elif init_type == 'xavier': init.xavier_normal(m.weight.data, gain=gain) elif init_type == 'kaiming': init.kaiming_normal(m.weight.data, a=0, mode='fan_in') elif init_type == 'orthogonal': init.orthogonal(m.weight.data, gain=gain) else: raise NotImplementedError('initialization method [%s] is not implemented' % init_type) if hasattr(m, 'bias') and m.bias is not None: init.constant(m.bias.data, 0.0) elif classname.find('BatchNorm2d') != -1: init.normal(m.weight.data, 1.0, gain) init.constant(m.bias.data, 0.0)
def init_fun(m): classname = m.__class__.__name__ if (classname.find('Conv') == 0 or classname.find('Linear') == 0) and hasattr(m, 'weight'): # print m.__class__.__name__ if init_type == 'gaussian': init.normal(m.weight.data, 0.0, 0.02) elif init_type == 'xavier': init.xavier_normal(m.weight.data, gain=math.sqrt(2)) elif init_type == 'kaiming': init.kaiming_normal(m.weight.data, a=0, mode='fan_in') elif init_type == 'orthogonal': init.orthogonal(m.weight.data, gain=math.sqrt(2)) elif init_type == 'default': pass else: assert 0, "Unsupported initialization: {}".format(init_type) if hasattr(m, 'bias') and m.bias is not None: init.constant(m.bias.data, 0.0)
def __init__(self, depth, pretrained=True, cut_at_pooling=False, num_features=0, norm=False, dropout=0, num_classes=0): super(ResNet, self).__init__() self.depth = depth self.pretrained = pretrained self.cut_at_pooling = cut_at_pooling # Construct base (pretrained) resnet if depth not in ResNet.__factory: raise KeyError("Unsupported depth:", depth) self.base = ResNet.__factory[depth](pretrained=pretrained) if not self.cut_at_pooling: self.num_features = num_features self.norm = norm self.dropout = dropout self.has_embedding = num_features > 0 self.num_classes = num_classes out_planes = self.base.fc.in_features # Append new layers if self.has_embedding: self.feat = nn.Linear(out_planes, self.num_features) self.feat_bn = nn.BatchNorm1d(self.num_features) init.kaiming_normal(self.feat.weight, mode='fan_out') init.constant(self.feat.bias, 0) init.constant(self.feat_bn.weight, 1) init.constant(self.feat_bn.bias, 0) else: # Change the num_features to CNN output channels self.num_features = out_planes if self.dropout > 0: self.drop = nn.Dropout(self.dropout) if self.num_classes > 0: self.classifier = nn.Linear(self.num_features, self.num_classes) init.normal(self.classifier.weight, std=0.001) init.constant(self.classifier.bias, 0) if not self.pretrained: self.reset_params()
def build_embedding(ssc_voc, ssc_vec, line, emb_voc, emb_vec): global oov, voc tokens = line.split() for i in range(len(tokens)): if tokens[i] not in ssc_voc: voc += 1 ssc_voc.append(tokens[i]) if tokens[i] in emb_voc: ssc_vec[tokens[i]] = torch.from_numpy(emb_vec[tokens[i]]).view(1, args.word_dim) else: ssc_vec[tokens[i]] = init.normal(torch.Tensor(1, args.word_dim), 0, args.init_weight) oov += 1 return ssc_voc, ssc_vec
def __init__(self, inputLen): super(XUANet, self).__init__() self.conv1 = nn.Conv2d(2, 2, kernel_size=1, stride=1) init.xavier_uniform(self.conv1.weight, gain=nn.init.calculate_gain('relu')) self.conv2 = nn.Conv2d(2, 1, kernel_size=1, stride=1) init.xavier_uniform(self.conv2.weight, gain=nn.init.calculate_gain('relu')) self.fc1 = nn.Linear(inputLen, 4096) init.normal(self.fc1.weight, mean=0., std=0.01) self.fc2 = nn.Linear(4096, 64) init.normal(self.fc2.weight, mean=0., std=0.01) self.fc3 = nn.Linear(64, 1) init.normal(self.fc3.weight, mean=0., std=0.01) # self.fc3.bias.data.copy_(torch.from_numpy(np.array([60.]))) # print('bias', self.fc3.bias) self.ceriation = nn.L1Loss()
def reset_parameters(self, init_fc_w=xavier_uniform, init_fc_b=lambda x: constant(x, 0), init_embed=lambda x: normal(x, std=0.05), **kwargs): """Resets the trainable weights.""" def set_constant_row(parameters, iRow=0, value=0): """Return `parameters` with row `iRow` as s constant `value`.""" data = parameters.data data[iRow, :] = value return torch.nn.Parameter(data, requires_grad=parameters.requires_grad) np.random.seed(self.seed) if self.seed is not None: torch.manual_seed(self.seed) if not self.isHash: self.embedding.weight = init_embed(self.embedding.weight) if self.paddingIdx is not None: # Unfortunately has to set weight to 0 even when paddingIdx =0 self.embedding.weight = set_constant_row(self.embedding.weight) else: self.embedding.reset_parameters(**kwargs) self.fc1.weight = init_fc_w(self.fc1.weight) self.fc1.biais = init_fc_b(self.fc1.weight) self.fc2.weight = init_fc_w(self.fc2.weight) self.fc2.biais = init_fc_b(self.fc2.weight) self.fc3.weight = init_fc_w(self.fc3.weight) self.fc3.biais = init_fc_b(self.fc3.weight) self.fc4.weight = init_fc_w(self.fc4.weight) self.fc4.biais = init_fc_b(self.fc4.weight)
def __init__(self, attn_input_size=128, attn_hidden_size=256, decoder_output_size=240, decoder_hidden_size=256, decoder_num_layers=2, max_length=30, use_cuda=False): super(AttnDecoder, self).__init__() self.attn_input_size = attn_input_size self.attn_hidden_size = attn_hidden_size self.decoder_hidden_size = decoder_hidden_size self.decoder_output_size = decoder_output_size self.decoder_num_layers = decoder_num_layers self.max_length = max_length self.use_cuda = use_cuda # initialize gru cells self.attn_gru = nn.GRUCell(self.attn_input_size, self.attn_hidden_size) self.decoder_grus = [] for i in range(self.decoder_num_layers): self.decoder_grus.append( nn.GRUCell(self.decoder_hidden_size, self.decoder_hidden_size)) # initialize weights self.w1 = Variable( torch.Tensor(self.attn_hidden_size, self.attn_hidden_size)) # https://github.com/pytorch/pytorch/blob/master/torch/nn/init.py init.normal(self.w1) self.w2 = Variable( torch.Tensor(self.attn_hidden_size, self.attn_hidden_size)) init.normal(self.w2) self.v = Variable(torch.Tensor(self.attn_hidden_size)) init.normal(self.v) # initialize other layers self.attn_combine = nn.Linear(self.attn_hidden_size * 2, self.attn_hidden_size) self.out = nn.Linear(self.decoder_hidden_size, self.decoder_output_size)
def reset_parameters(self): """ Initialize parameters following the way proposed in the paper. """ init.orthogonal(self.weight_ih.data) weight_hh_data = torch.eye(self.hidden_size) weight_hh_data = weight_hh_data.repeat(1, 4) self.weight_hh.data.set_(weight_hh_data) # weight_ch_data = torch.eye(self.hidden_size) # weight_ch_data = weight_ch_data.repeat(1, 4) # self.weight_ch.data.set_(weight_ch_data) # init.constant(self.weight_ch_i, val=0) # init.constant(self.weight_ch_f, val=0) # init.constant(self.weight_ch_o, val=0) init.normal(self.weight_ch_i) init.normal(self.weight_ch_f) init.normal(self.weight_ch_o) # The bias is just set to zero vectors. if self.use_bias: init.constant(self.bias.data, val=0)
def __init__(self, n_inner_layers, input_dim, hidden_dim, output_dim, dropout=0.05): super(SELUNet, self).__init__() self.dropout = dropout self.input_dim = input_dim self.output_dim = output_dim self.hidden_dim = hidden_dim self.n_inner_layers = n_inner_layers self.fc_in = nn.Linear(input_dim, hidden_dim) for k in range(n_inner_layers): setattr(self, "fc_%s" % k, nn.Linear(hidden_dim, hidden_dim)) self.fc_out = nn.Linear(hidden_dim, output_dim) # Initialize weights specifically for selu # First layer init.normal(self.fc_in.weight, std=1. / np.sqrt(np.float32(self.input_dim))) init.constant(self.fc_in.bias, 0.) # Inner layers for i in range(self.n_inner_layers): init.normal(getattr(self, "fc_%s" % i).weight, std=1. / np.sqrt(np.float32(self.hidden_dim))) init.constant(getattr(self, "fc_%s" % i).bias, 0.) # Last layer init.normal(self.fc_out.weight, std=1. / np.sqrt(np.float32(self.hidden_dim))) init.constant(self.fc_out.bias, 0.)
def __init__(self, nClasses): super(SqueezeNet, self).__init__() self.num_classes = nClasses self.conv1 = nn.Conv2d(1, 96, kernel_size=7, stride=2) self.bn1 = nn.BatchNorm2d(96) self.relu1 = nn.ReLU(inplace=True) self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True) self.conv21 = nn.Conv2d(96, 16, kernel_size=1) self.bn21 = nn.BatchNorm2d(16) self.relu21 = nn.ReLU(inplace=True) self.conv22 = nn.Conv2d(16, 64, kernel_size=1) self.bn22 = nn.BatchNorm2d(64) self.relu22 = nn.ReLU(inplace=True) self.conv23 = nn.Conv2d(16, 64, kernel_size=3, padding=1) self.bn23 = nn.BatchNorm2d(64) self.relu23 = nn.ReLU(inplace=True) self.conv31 = nn.Conv2d(128, 16, kernel_size=1, bias=False) self.bn31 = nn.BatchNorm2d(16) self.relu31 = nn.ReLU(inplace=True) self.bn32 = nn.BatchNorm2d(16) self.activ32 = Active() self.conv32 = nn.Conv2d(16, 64, kernel_size=1, bias=False) self.relu32 = nn.ReLU(inplace=True) self.bn33 = nn.BatchNorm2d(16) self.activ33 = Active() self.conv33 = nn.Conv2d(16, 64, kernel_size=3, padding=1, bias=False) self.relu33 = nn.ReLU(inplace=True) self.conv41 = nn.Conv2d(128, 32, kernel_size=1, bias=False) self.bn41 = nn.BatchNorm2d(32) self.relu41 = nn.ReLU(inplace=True) self.bn42 = nn.BatchNorm2d(32) self.activ42 = Active() self.conv42 = nn.Conv2d(32, 128, kernel_size=1, bias=False) self.relu42 = nn.ReLU(inplace=True) self.bn43 = nn.BatchNorm2d(32) self.activ43 = Active() self.conv43 = nn.Conv2d(32, 128, kernel_size=3, padding=1, bias=False) self.relu43 = nn.ReLU(inplace=True) self.maxpool5 = nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True) self.conv61 = nn.Conv2d(256, 32, kernel_size=1, bias=False) self.bn61 = nn.BatchNorm2d(32) self.relu61 = nn.ReLU(inplace=True) self.bn62 = nn.BatchNorm2d(32) self.activ62 = Active() self.conv62 = nn.Conv2d(32, 128, kernel_size=1, bias=False) self.relu62 = nn.ReLU(inplace=True) self.bn63 = nn.BatchNorm2d(32) self.activ63 = Active() self.conv63 = nn.Conv2d(32, 128, kernel_size=3, padding=1, bias=False) self.relu63 = nn.ReLU(inplace=True) self.conv71 = nn.Conv2d(256, 48, kernel_size=1, bias=False) self.bn71 = nn.BatchNorm2d(48) self.relu71 = nn.ReLU(inplace=True) self.bn72 = nn.BatchNorm2d(48) self.activ72 = Active() self.conv72 = nn.Conv2d(48, 192, kernel_size=1, bias=False) self.relu72 = nn.ReLU(inplace=True) self.bn73 = nn.BatchNorm2d(48) self.activ73 = Active() self.conv73 = nn.Conv2d(48, 192, kernel_size=3, padding=1, bias=False) self.relu73 = nn.ReLU(inplace=True) self.conv81 = nn.Conv2d(384, 48, kernel_size=1, bias=False) self.bn81 = nn.BatchNorm2d(48) self.relu81 = nn.ReLU(inplace=True) self.bn82 = nn.BatchNorm2d(48) self.activ82 = Active() self.conv82 = nn.Conv2d(48, 192, kernel_size=1, bias=False) self.relu82 = nn.ReLU(inplace=True) self.bn83 = nn.BatchNorm2d(48) self.activ83 = Active() self.conv83 = nn.Conv2d(48, 192, kernel_size=3, padding=1, bias=False) self.relu83 = nn.ReLU(inplace=True) self.conv91 = nn.Conv2d(384, 64, kernel_size=1, bias=False) self.bn91 = nn.BatchNorm2d(64) self.relu91 = nn.ReLU(inplace=True) self.bn92 = nn.BatchNorm2d(64) self.activ92 = Active() self.conv92 = nn.Conv2d(64, 256, kernel_size=1, bias=False) self.relu92 = nn.ReLU(inplace=True) self.bn93 = nn.BatchNorm2d(64) self.activ93 = Active() self.conv93 = nn.Conv2d(64, 256, kernel_size=3, padding=1, bias=False) self.relu93 = nn.ReLU(inplace=True) self.maxpool10 = nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True) self.conv111 = nn.Conv2d(512, 64, kernel_size=1, bias=False) self.bn111 = nn.BatchNorm2d(64) self.relu111 = nn.ReLU(inplace=True) self.conv112 = nn.Conv2d(64, 256, kernel_size=1, bias=False) self.bn112 = nn.BatchNorm2d(256) self.relu112 = nn.ReLU(inplace=True) self.conv113 = nn.Conv2d(64, 256, kernel_size=3, padding=1, bias=False) self.bn113 = nn.BatchNorm2d(256) self.relu113 = nn.ReLU(inplace=True) self.drop121 = nn.Dropout(p=0.2) self.conv121 = nn.Conv2d(512, self.num_classes, kernel_size=1) self.relu121 = nn.ReLU(inplace=True) self.avgpool121 = nn.AvgPool2d(13) for m in self.modules(): if isinstance(m, nn.Conv2d): if m is self.conv121: init.normal(m.weight.data, mean=0.0, std=0.01) else: init.kaiming_uniform(m.weight.data) if m.bias is not None: m.bias.data.zero_()
def __init__(self,block,layers, depth, pretrained=True, cut_at_pooling=False, num_features=0, norm=False, dropout=0, num_classes=0, FCN=False, radius=1., thresh=0.5): super(ResNet, self).__init__() self.depth = depth self.pretrained = pretrained self.cut_at_pooling = cut_at_pooling self.FCN = FCN self.inplanes = 64 super(ResNet, self).__init__() self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = nn.BatchNorm2d(64) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) self.layer3 = self._make_layer(block, 256, layers[2], stride=2) self.layer4 = self._make_layer(block, 512, layers[3], stride=1) # ==========================add dilation=============================# if self.FCN: self.num_features = num_features self.num_classes = 751 # num_classes self.dropout = dropout # out_planes = self.base.fc.in_features self.local_conv1 = nn.Conv2d(256, self.num_features, kernel_size=1, padding=0, bias=False) self.local_conv2 = nn.Conv2d(512, self.num_features, kernel_size=1, padding=0, bias=False) self.local_conv3 = nn.Conv2d(1024, self.num_features, kernel_size=1, padding=0, bias=False) self.local_conv4 = nn.Conv2d(2048, self.num_features, kernel_size=1, padding=0, bias=False) init.kaiming_normal(self.local_conv1.weight, mode='fan_out') init.kaiming_normal(self.local_conv2.weight, mode='fan_out') init.kaiming_normal(self.local_conv3.weight, mode='fan_out') init.kaiming_normal(self.local_conv4.weight, mode='fan_out') # init.constant(self.local_conv.bias,0) self.feat_bn2d = nn.BatchNorm2d(self.num_features) # may not be used, not working on caffe init.constant(self.feat_bn2d.weight, 1) # initialize BN, may not be used init.constant(self.feat_bn2d.bias, 0) # iniitialize BN, may not be used # self.offset = ConvOffset2D(256) ##---------------------------stripe1----------------------------------------------# self.instance0 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance0.weight, std=0.001) init.constant(self.instance0.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance1 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance1.weight, std=0.001) init.constant(self.instance1.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance2 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance2.weight, std=0.001) init.constant(self.instance2.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance3 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance3.weight, std=0.001) init.constant(self.instance3.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance4 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance4.weight, std=0.001) init.constant(self.instance4.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance5 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance5.weight, std=0.001) init.constant(self.instance5.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance6 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance5.weight, std=0.001) init.constant(self.instance5.bias, 0) self.drop = nn.Dropout(self.dropout) if not self.pretrained: self.reset_params()
def reset_parameters(self): normal(self.weight.data) normal(self.noise.data) normal(self.bias.data)
def weights_init_classifier(m): classname = m.__class__.__name__ if classname.find('Linear') != -1: init.normal(m.weight.data, std=0.001) init.constant(m.bias.data, 0.0)
def _init_weight(self): # init conv1 init.normal(self.conv1.weight, std=0.01) # init.constant(self.conv1.bias, 0) # init conv2 init.normal(self.conv2.weight, std=0.01)
def __init__(self, args, output_channels=40): super(DGCNN_Transformer, self).__init__() self.args = args self.k = args.k self.deg = args.deg self.delta = args.delta self.neighbors = args.neighbors self.rpe = args.rpe self.ape = args.ape self.scale = args.scale print("self.deg : {}".format(self.deg)) print("self.delta : {}".format(self.delta)) print("self.neighbors : {}".format(self.neighbors)) print("self.rpe : {}".format(self.rpe)) print("self.ape : {}".format(self.ape)) self.bn1 = nn.BatchNorm2d(64) self.bn2 = nn.BatchNorm2d(64) self.bn3 = nn.BatchNorm2d(128) self.bn4 = nn.BatchNorm2d(256) self.bn5 = nn.BatchNorm1d(args.emb_dims) if self.deg: self.deg_bn1 = nn.BatchNorm1d(1024) self.deg_bn2 = nn.BatchNorm1d(1024) self.deg_bn3 = nn.BatchNorm1d(1024) self.deg_bn4 = nn.BatchNorm1d(1024) else: self.deg_bn1 = None self.deg_bn2 = None self.deg_bn3 = None self.deg_bn4 = None if self.ape: # self.pos_nn1 is skipped self.pos_nn2 = nn.Sequential( nn.Conv2d(3, 32, kernel_size=1, bias=False), nn.BatchNorm2d(32), nn.Tanh(), nn.Conv2d(32, 64, kernel_size=1, bias=False), nn.BatchNorm2d(64)) # B, 3, N, 1 -> B, C, N, 1 self.pos_nn3 = nn.Sequential( nn.Conv2d(3, 32, kernel_size=1, bias=False), nn.BatchNorm2d(32), nn.Tanh(), nn.Conv2d(32, 64, kernel_size=1, bias=False), nn.BatchNorm2d(64)) # B, 3, N, 1 -> B, C, N, 1 self.pos_nn4 = nn.Sequential( nn.Conv2d(3, 32, kernel_size=1, bias=False), nn.BatchNorm2d(32), nn.Tanh(), nn.Conv2d(32, 128, kernel_size=1, bias=False), nn.BatchNorm2d(128)) # B, 3, N, 1 -> B, C, N, 1 else: self.pos_nn2 = None self.pos_nn3 = None self.pos_nn4 = None # deg : False # neighbors : True # delta : True self.conv1 = AttentionConv(3 * 2, 64, kernel_size=self.k, groups=8, rpe=self.rpe, scale=self.scale, return_kv=True, layer=1) self.act1 = nn.LeakyReLU(negative_slope=0.2) self.conv2 = AttentionConv(64 * 2, 64, kernel_size=self.k, groups=8, rpe=self.rpe, scale=self.scale, return_kv=True, layer=2) self.act2 = nn.LeakyReLU(negative_slope=0.2) self.conv3 = AttentionConv(64 * 2, 128, kernel_size=self.k, groups=8, rpe=self.rpe, scale=self.scale, return_kv=True, layer=3) self.act3 = nn.LeakyReLU(negative_slope=0.2) self.conv4 = AttentionConv(128 * 2, 256, kernel_size=self.k, groups=8, rpe=self.rpe, scale=self.scale, return_kv=True, layer=4) self.act4 = nn.LeakyReLU(negative_slope=0.2) self.conv5 = nn.Sequential( nn.Conv1d(256, args.emb_dims, kernel_size=1, bias=False), self.bn5, nn.LeakyReLU(negative_slope=0.2)) # 64 + 64 + 128 + 256 = 512 self.linear1 = nn.Linear(args.emb_dims, 512, bias=False) self.bn6 = nn.BatchNorm1d(512) self.dp1 = nn.Dropout(p=args.dropout) self.linear2 = nn.Linear(512, 256) self.bn7 = nn.BatchNorm1d(256) self.dp2 = nn.Dropout(p=args.dropout) self.linear3 = nn.Linear(256, output_channels) for key in self.state_dict(): if key.split('.')[-1] == "weight": try: if "conv" in key: init.kaiming_normal(self.state_dict()[key]) except: init.normal(self.state_dict()[key]) if "bn" in key: self.state_dict()[key][...] = 1 elif key.split(".")[-1] == 'bias': self.state_dict()[key][...] = 0
def weights_init(m): if isinstance(m, nn.Conv2d): init.normal(m.weight.data, std=0.01) m.bias.data.zero_()
def __init__(self, layers=10, blocks=4, dilation_channels=32, residual_channels=32, skip_channels=256, end_channels=256, classes=256, output_length=32, kernel_size=2, dtype=torch.FloatTensor, bias=False, args=None): super(WaveNetModel, self).__init__() self.layers = layers self.blocks = blocks self.dilation_channels = dilation_channels self.residual_channels = residual_channels self.skip_channels = skip_channels self.classes = classes self.kernel_size = kernel_size self.dtype = dtype # build model receptive_field = 1 init_dilation = 1 self.dilations = [] self.dilated_queues = [] # self.main_convs = nn.ModuleList() self.filter_convs = nn.ModuleList() self.gate_convs = nn.ModuleList() self.residual_convs = nn.ModuleList() self.skip_convs = nn.ModuleList() # 1x1 convolution to create channels self.start_conv = nn.Conv1d(in_channels=self.classes, out_channels=residual_channels, kernel_size=1, bias=bias) for b in range(blocks): additional_scope = kernel_size - 1 new_dilation = 1 for i in range(layers): # dilations of this layer self.dilations.append((new_dilation, init_dilation)) # dilated queues for fast generation self.dilated_queues.append( DilatedQueue(max_length=(kernel_size - 1) * new_dilation + 1, num_channels=residual_channels, dilation=new_dilation, dtype=dtype)) # dilated convolutions filter_temp_conv = nn.Conv1d(in_channels=residual_channels, out_channels=dilation_channels, kernel_size=kernel_size, bias=bias) if args.cs: init.normal(filter_temp_conv.weight, mean=0, std=args.gain_factor / math.sqrt(kernel_size)) self.filter_convs.append(filter_temp_conv) gate_temp_conv = nn.Conv1d(in_channels=residual_channels, out_channels=dilation_channels, kernel_size=kernel_size, bias=bias) if args.cs: init.constant(gate_temp_conv.weight, 0) self.gate_convs.append(gate_temp_conv) # 1x1 convolution for residual connection self.residual_convs.append( nn.Conv1d(in_channels=dilation_channels, out_channels=residual_channels, kernel_size=1, bias=bias)) # 1x1 convolution for skip connection self.skip_convs.append( nn.Conv1d(in_channels=dilation_channels, out_channels=skip_channels, kernel_size=1, bias=bias)) receptive_field += additional_scope additional_scope *= 2 init_dilation = new_dilation new_dilation *= 2 self.end_conv_1 = nn.Conv1d(in_channels=skip_channels, out_channels=end_channels, kernel_size=1, bias=True) self.end_conv_2 = nn.Conv1d(in_channels=end_channels, out_channels=classes, kernel_size=1, bias=True) # self.output_length = 2 ** (layers - 1) self.output_length = output_length self.receptive_field = receptive_field
def get_model(class_weights=None): model = resnet18(pretrained=True) # make all params untrainable for p in model.parameters(): p.requires_grad = False # reset the last fc layer model.fc = nn.Linear(512, 256) normal(model.fc.weight, 0.0, 0.01) constant(model.fc.bias, 0.0) # make some other params trainable trainable_params = [] trainable_params += [n for n, p in model.named_parameters() if 'layer4' in n or 'layer3' in n] for n, p in model.named_parameters(): if n in trainable_params: p.requires_grad = True for m in model.layer4.modules(): if isinstance(m, nn.ReLU): m.inplace = False for m in model.layer3.modules(): if isinstance(m, nn.ReLU): m.inplace = False # create different parameter groups classifier_weights = [model.fc.weight] classifier_biases = [model.fc.bias] features_weights = [ p for n, p in model.named_parameters() if n in trainable_params and 'conv' in n ] features_weights += [ p for n, p in model.named_parameters() if n in trainable_params and 'downsample.0' in n and 'weight' in n ] features_bn_weights = [ p for n, p in model.named_parameters() if n in trainable_params and 'weight' in n and ('bn' in n or 'downsample.1' in n) ] features_bn_biases = [ p for n, p in model.named_parameters() if n in trainable_params and 'bias' in n ] # you can set different learning rates classifier_lr = 1e-2 features_lr = 1e-2 # but they are not actually used (because lr_scheduler is used) params = [ {'params': classifier_weights, 'lr': classifier_lr, 'weight_decay': 1e-3}, {'params': classifier_biases, 'lr': classifier_lr}, {'params': features_weights, 'lr': features_lr, 'weight_decay': 1e-3}, {'params': features_bn_weights, 'lr': features_lr}, {'params': features_bn_biases, 'lr': features_lr} ] optimizer = optim.SGD(params, momentum=0.9, nesterov=True) # loss function criterion = nn.CrossEntropyLoss(weight=class_weights).cuda() # move the model to gpu model = model.cuda() return model, criterion, optimizer
# squeezing - 1크기 dimension 제거 x1 = torch.FloatTensor(10, 1, 3, 1, 4) x2 = torch.squeeze(x1) x1.size(), x2.size() # unsqueezing - 1크기 dimension 추가 x1 = torch.FloatTensor(10, 3, 4) x2 = torch.unsqueeze(x1, dim=0) x1.size(), x2.size() # 분포로부터 데이터 생성 import torch.nn.init as init init.uniform(torch.FloatTensor(3, 4), a=0, b=9) # a~b까지 uniform distribution에서 데이터 생성 init.normal(torch.FloatTensor(3, 4), mean=0, std=1) # 평균이 mean이고 표준편차가 std인 normal distribution에서 데이터 생성 init.constant(torch.FloatTensor(3, 4), val=5) # val 값으로 input 데이터 크기의 상수 생성 # 수학 연산 a = torch.FloatTensor([[1, 2, 3], [4, 5, 6]]) b = torch.FloatTensor([[1, 2, 3], [4, 5, 6]]) torch.add(a, b) torch.add(a, 10) # broadcasting a + b torch.mul(a, b) torch.mul(a, 10) # broadcasting a * b torch.div(a, b)
def __init__(self, num_classes, loss, block, layers, zero_init_residual=False, groups=1, width_per_group=64, replace_stride_with_dilation=None, norm_layer=None, last_stride=2, fc_dims=None, dropout_p=None, **kwargs): super(ResNet, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm2d self.hashbit = 256 self._norm_layer = norm_layer self.loss = loss self.feature_dim = 512 * block.expansion self.inplanes = 64 self.dilation = 1 if replace_stride_with_dilation is None: # each element in the tuple indicates if we should replace # the 2x2 stride with a dilated convolution instead replace_stride_with_dilation = [False, False, False] if len(replace_stride_with_dilation) != 3: raise ValueError("replace_stride_with_dilation should be None " "or a 3-element tuple, got {}".format( replace_stride_with_dilation)) self.groups = groups self.base_width = width_per_group self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = norm_layer(self.inplanes) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2, dilate=replace_stride_with_dilation[0]) self.layer3 = self._make_layer(block, 256, layers[2], stride=2, dilate=replace_stride_with_dilation[1]) # print(last_stride) self.layer4 = self._make_layer(block, 512, layers[3], stride=last_stride, dilate=replace_stride_with_dilation[2]) # self.global_avgpool = nn.AdaptiveAvgPool2d((1, 1)) # self.fc = self._construct_fc_layer( # fc_dims, 512 * block.expansion, dropout_p # ) self.global_avgpool = nn.AdaptiveAvgPool2d(1) assert fc_dims is not None self.fc_fusion = self._construct_fc_layer(fc_dims, 512 * block.expansion * 2) self.feature_dim += 512 * block.expansion self.classifier = nn.Linear(self.hashbit, num_classes) self.classifier2 = nn.Linear(self.hashbit, num_classes) #self._init_params() # if(self.hashbit<=128): # self.fc_encode = nn.Linear(self.feature_dim, self.hashbit) # self.classifier = nn.Linear(self.hashbit, num_classes) # else: # self.fc_encode = nn.Linear(self.feature_dim, self.hashbit//2) # self.classifier = nn.Linear(self.hashbit//2, num_classes) # self.fc_encode = nn.Linear(self.feature_dim, 128) self.layer4_pcb = self._make_layer_pcb(block, 512, layers[3], stride=1) num_stripes = 6 local_conv_out_channels = 256 self.layer_nums = 3 self.num_stripes = 6 self.local_conv_list = nn.ModuleList() for _ in range(3 * self.num_stripes - 3): self.local_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) if num_classes > 0: self.fc_list = nn.ModuleList() for _ in range(3 * num_stripes - 3): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal(fc.weight, std=0.001) init.constant(fc.bias, 0) self.fc_list.append(fc) # if(self.hashbit<=128): # self.fc_encode_pcb = nn.Linear(local_conv_out_channels*15, self.hashbit) # else: # self.fc_encode_pcb = nn.Linear(local_conv_out_channels*15, self.hashbit-self.hashbit//2) self.split_num = 10 self.fc_split = nn.Linear( local_conv_out_channels * 15 + self.feature_dim, self.hashbit * self.split_num) self.de1 = DivideEncode(self.hashbit * self.split_num, self.split_num) # Zero-initialize the last BN in each residual branch, # so that the residual branch starts with zeros, and each residual block behaves like an identity. # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 if zero_init_residual: for m in self.modules(): if isinstance(m, Bottleneck): nn.init.constant_(m.bn3.weight, 0) elif isinstance(m, BasicBlock): nn.init.constant_(m.bn2.weight, 0)
def _set_init(self, layer): init.normal(layer.weight, mean=0, std=1)
def __init__(self, depth, pretrained=True, cut_at_pooling=False, num_features=0, norm=False, dropout=0, num_classes=0, FCN=False, T=1, dim=256): super(ResNet, self).__init__() self.depth = depth self.pretrained = pretrained self.cut_at_pooling = cut_at_pooling self.FCN = FCN self.T = T self.reduce_dim = dim # self.offset = ConvOffset2D(32) # Construct base (pretrained) resnet if depth not in ResNet.__factory: raise KeyError("Unsupported depth:", depth) self.base = ResNet.__factory[depth](pretrained=pretrained) #==========================add dilation=============================# if self.FCN: self.base.layer4[0].conv2.stride = (1, 1) # self.base.layer4[0].conv2.dilation=(2,2) # self.base.layer4[0].conv2.padding=(2,2) self.base.layer4[0].downsample[0].stride = (1, 1) # self.base.layer4[1].conv2.dilation=(2,2) # self.base.layer4[1].conv2.padding=(2,2) # self.base.layer4[2].conv2.dilation=(2,2) # self.base.layer4[2].conv2.padding=(2,2) #================append conv for FCN==============================# self.num_features = num_features self.num_classes = num_classes self.dropout = dropout self.local_conv = nn.Conv2d(2048, self.num_features, kernel_size=1, padding=0, bias=False) init.kaiming_normal(self.local_conv.weight, mode='fan_out') # init.constant(self.local_conv.bias,0) self.feat_bn2d = nn.BatchNorm2d( self.num_features) #may not be used, not working on caffe init.constant(self.feat_bn2d.weight, 1) #initialize BN, may not be used init.constant(self.feat_bn2d.bias, 0) # iniitialize BN, may not be used ##---------------------------stripe1----------------------------------------------# self.instance0 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance0.weight, std=0.001) init.constant(self.instance0.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance1 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance1.weight, std=0.001) init.constant(self.instance1.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance2 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance2.weight, std=0.001) init.constant(self.instance2.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance3 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance3.weight, std=0.001) init.constant(self.instance3.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance4 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance4.weight, std=0.001) init.constant(self.instance4.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.instance5 = nn.Linear(self.num_features, self.num_classes) init.normal(self.instance5.weight, std=0.001) init.constant(self.instance5.bias, 0) ##---------------------------stripe1----------------------------------------------# ##---------------------------stripe1----------------------------------------------# self.drop = nn.Dropout(self.dropout) self.local_mask = nn.Conv2d(self.reduce_dim, 7, kernel_size=1, padding=0, bias=True) init.kaiming_normal(self.local_mask.weight, mode='fan_out') # init.xavier_normal(self.local_mask.weight) init.constant(self.local_mask.bias, 0) #===================================================================# elif not self.cut_at_pooling: self.num_features = num_features self.norm = norm self.dropout = dropout self.has_embedding = num_features > 0 self.num_classes = num_classes out_planes = self.base.fc.in_features # Append new layers if self.has_embedding: # self.f_bn = nn.BatchNorm1d(2048) # init.constant(self.f_bn.weight, 1) # init.constant(self.f_bn.bias, 0) self.feat = nn.Linear(out_planes, self.num_features, bias=False) self.feat_bn = nn.BatchNorm1d(self.num_features) init.kaiming_normal(self.feat.weight, mode='fan_out') # init.constant(self.feat.bias, 0) init.constant(self.feat_bn.weight, 1) init.constant(self.feat_bn.bias, 0) else: # Change the num_features to CNN output channels self.num_features = out_planes if self.dropout > 0: self.drop = nn.Dropout(self.dropout) if self.num_classes > 0: # self.classifier = nn.Linear(self.num_features, self.num_classes) self.classifier = nn.Linear(self.num_features, self.num_classes) init.normal(self.classifier.weight, std=0.001) init.constant(self.classifier.bias, 0) if not self.pretrained: self.reset_params()
third_seg_map = self.upsample_layer(third_seg_map) second_seg_map = torch.add(third_seg_map, second_seg_map) second_seg_map = self.upsample_layer(second_seg_map) x = torch.add(first_seg_map, second_seg_map) return x net = Unet() net.cuda(1) prev_time = time.clock() for param in net.parameters(): try: nout = param.size()[0] nin = param.size()[1] ini.normal(param.data, mean=0, std=0.01) param = param / ((2 / (nin + nout))**0.5) except: pass # In[2]: f = h5py.File('Unet-training.h5') SAMPLE = [ "LG/0001", "LG/0002", "LG/0004", "LG/0006", "LG/0008", "LG/0011", "LG/0012", "LG/0013", "LG/0014", "LG/0015", "HG/0001", "HG/0002", "HG/0003", "HG/0004", "HG/0005", "HG/0006", "HG/0007", "HG/0008", "HG/0009", "HG/0010", "HG/0011", "HG/0012", "HG/0013", "HG/0014", "HG/0015", "HG/0022", "HG/0024", "HG/0025", "HG/0026" ]
def _set_init(self, layer): init.normal(layer.weight, mean=0., std=.1) init.constant(layer.bias, B_INIT)
def gaussian_intiailize(model, std=.1): for p in model.parameters(): init.normal(p, std=std)
def __init__(self, args, output_channels=40): super(DGCNN_Transformer, self).__init__() self.args = args self.k = args.k self.ape = args.ape self.scale = args.scale print("self.ape : {}".format(self.ape)) print("self.scale : {}".format(self.scale)) self.bn1 = nn.BatchNorm2d(64) self.bn2 = nn.BatchNorm2d(64) self.bn3 = nn.BatchNorm2d(128) self.bn4 = nn.BatchNorm2d(256) self.bn5 = nn.BatchNorm1d(args.emb_dims) self.conv1 = AttentionConv(3 * 2, 64, kernel_size=self.k, groups=8, ape=self.ape, scale=self.scale) self.act1 = nn.LeakyReLU(negative_slope=0.2) self.conv2 = AttentionConv(64 * 2, 64, kernel_size=self.k, groups=8, ape=self.ape, scale=self.scale) self.act2 = nn.LeakyReLU(negative_slope=0.2) self.conv3 = AttentionConv(64 * 2, 128, kernel_size=self.k, groups=8, ape=self.ape, scale=self.scale) self.act3 = nn.LeakyReLU(negative_slope=0.2) self.conv4 = AttentionConv(128 * 2, 256, kernel_size=self.k, groups=8, ape=self.ape, scale=self.scale) self.act4 = nn.LeakyReLU(negative_slope=0.2) self.conv5 = nn.Sequential( nn.Conv1d(256, args.emb_dims, kernel_size=1, bias=False), self.bn5, nn.LeakyReLU(negative_slope=0.2)) # 64 + 64 + 128 + 256 = 512 self.linear1 = nn.Linear(args.emb_dims, 512, bias=False) self.bn6 = nn.BatchNorm1d(512) self.dp1 = nn.Dropout(p=args.dropout) self.linear2 = nn.Linear(512, 256) self.bn7 = nn.BatchNorm1d(256) self.dp2 = nn.Dropout(p=args.dropout) self.linear3 = nn.Linear(256, output_channels) for key in self.state_dict(): if key.split('.')[-1] == "weight": try: if "conv" in key: init.kaiming_normal(self.state_dict()[key]) except: init.normal(self.state_dict()[key]) if "bn" in key: self.state_dict()[key][...] = 1 elif key.split(".")[-1] == 'bias': self.state_dict()[key][...] = 0
import torch import torch.nn as nn import torch.optim as optim import torch.nn.init as init from torch.autograd import Variable from visdom import Visdom viz = Visdom() num_data=1000 num_epoch=400 x = init.uniform(torch.Tensor(num_data,1),-10,10) y = init.uniform(torch.Tensor(num_data,1),-10,10) z = x**2 + y**2 x_noise = x + init.normal(torch.FloatTensor(num_data,1),std=0.5) y_noise = y + init.normal(torch.FloatTensor(num_data,1),std=0.5) z_noise = x_noise**2 + y_noise**2 data_noise = torch.cat([x_noise,y_noise,z_noise],1) # visualize data win_1=viz.scatter( X=data_noise, opts=dict( markersize=5, markercolor=np.ndarray(shape=[num_data,3],dtype=float,buffer=[51,153,255]*np.ones(shape=[num_data,3])) ) ) model = nn.Sequential(
def __init__( self, model_path='tf_model_zoo/ECO/ECO.yaml', num_classes=101, weight_url='http://pa0630vji.bkt.gdipper.com/zhangcan/pth/models/kin_ECO_rgb_checkpoint_epoch_5-c4b5bd6c.pth.tar', num_segments=4): super(ECO, self).__init__() self.num_segments = num_segments manifest = yaml.load(open(model_path)) layers = manifest['layers'] self._channel_dict = dict() self._op_list = list() for l in layers: out_var, op, in_var = parse_expr(l['expr']) if op != 'Concat' and op != 'Eltwise': id, out_name, module, out_channel, in_name = get_basic_layer( l, 3 if len(self._channel_dict) == 0 else self._channel_dict[in_var[0]], conv_bias=False if op == 'Conv3d' else True) self._channel_dict[out_name] = out_channel setattr(self, id, module) self._op_list.append((id, op, out_name, in_name)) elif op == 'Concat': self._op_list.append((id, op, out_var[0], in_var)) channel = sum([self._channel_dict[x] for x in in_var]) self._channel_dict[out_var[0]] = channel else: self._op_list.append((id, op, out_var[0], in_var)) channel = self._channel_dict[in_var[0]] self._channel_dict[out_var[0]] = channel # load pretrained model on other dataset model_dict = self.state_dict() pretrained_on_kin = torch.utils.model_zoo.load_url(weight_url) new_state_dict = { k[18:]: v for k, v in pretrained_on_kin['state_dict'].items() if k[18:] in model_dict } # init the layer names which is not in pretrained model dict un_init_dict_keys = [ k for k in model_dict.keys() if k not in new_state_dict ] print(un_init_dict_keys) std = 0.001 for k in un_init_dict_keys: new_state_dict[k] = torch.DoubleTensor( model_dict[k].size()).zero_() if 'weight' in k: normal(new_state_dict[k], 0, std) elif 'bias' in k: constant(new_state_dict[k], 0) self.load_state_dict(new_state_dict)
def __init__(self, n_funcs=128, n_samples=10, stddev=0.01, x_dim=2, calc_true_params=False, use_cuda=False): self.use_cuda = use_cuda self.stddev = stddev self.num_of_funcs = n_funcs self.n_samples = n_samples self.xdim = x_dim true_params = torch.FloatTensor(n_funcs, x_dim) self.true_params = Variable(init.uniform(true_params), requires_grad=False) if self.use_cuda: params = torch.FloatTensor(n_funcs, x_dim).cuda() self.params = Variable(init.normal(params, mean=0., std=stddev).cuda(), requires_grad=True) self.true_params = self.true_params.cuda() else: params = torch.FloatTensor(n_funcs, x_dim) self.params = Variable(init.normal(params, mean=0., std=stddev), requires_grad=True) self.initial_params = self.params.clone() w = torch.FloatTensor(self.num_of_funcs, self.n_samples, self.xdim) self.W = Variable(init.uniform(w), requires_grad=False) if self.use_cuda: self.W = self.W.cuda() self.y_no_noise = torch.squeeze( torch.bmm(self.W, self.true_params.unsqueeze(self.true_params.dim()))) if self.num_of_funcs == 1: self.y_no_noise = self.y_no_noise.unsqueeze(0) noise = torch.FloatTensor(self.num_of_funcs, self.n_samples) self.noise = Variable(init.normal(noise, std=stddev), requires_grad=False) true_minimum_nll = np.array( [0.5 * self.n_samples * (np.log(stddev) + np.log(2 * np.pi))]) self.true_minimum_nll = Variable( torch.from_numpy(true_minimum_nll).float()) if self.use_cuda: self.noise = self.noise.cuda() self.true_minimum_nll = self.true_minimum_nll.cuda() self.y = self.y_no_noise + self.noise.expand_as(self.y_no_noise) # new_version # y = torch.FloatTensor(self.num_of_funcs, self.n_samples) # self.y = Variable(init.uniform(y), requires_grad=False) if self.use_cuda: self.y = self.y.cuda() if calc_true_params: self.true_params = self.get_true_params() self.param_hist = {} self._add_param_values(self.initial_params) self.nll_init = neg_log_likelihood_loss(self.y, self.y_t(self.initial_params), self.stddev, N=self.n_samples, avg_batch=False, size_average=False) self.nll_min = neg_log_likelihood_loss(self.y_no_noise, self.y_t(self.true_params), self.stddev, N=self.n_samples, avg_batch=False, size_average=False) self.distance_to_min = (self.nll_init - self.nll_min).data.cpu().squeeze().numpy()
data_time_steps = np.linspace(2, 10, seq_length + 1) data = np.sin(data_time_steps) data.resize((seq_length + 1, 1)) x = Variable(torch.Tensor(data[:-1]).type(dtype), requires_grad=False) y = Variable(torch.Tensor(data[1:]).type(dtype), requires_grad=False) data_time_steps = np.linspace(2, 10, seq_length + 1) data = np.sin(data_time_steps) data.resize((seq_length + 1, 1)) x = Variable(torch.Tensor(data[:-1]).type(dtype), requires_grad=False) y = Variable(torch.Tensor(data[1:]).type(dtype), requires_grad=False) w1 = torch.FloatTensor(input_size, hidden_size).type(dtype) init.normal(w1, 0.0, 0.4) w1 = Variable(w1, requires_grad=True) w2 = torch.FloatTensor(hidden_size, output_size).type(dtype) init.normal(w2, 0.0, 0.3) w2 = Variable(w2, requires_grad=True) def forward(input, context_state, w1, w2): xh = torch.cat((input, context_state), 1) context_state = torch.tanh(xh.mm(w1)) out = context_state.mm(w2) return (out, context_state) for i in range(epochs): total_loss = 0 context_state = Variable(torch.zeros((1, hidden_size)).type(dtype), requires_grad=True) for j in range(x.size(0)):
def show_image(img_2d, cmap='gray'): plt.imshow(img_2d, cmap=cmap) plt.show() print('PyTorch version = {}'.format(torch.__version__)) # 1.data generation # option : 정규화 std = 5 num_data = 1000 # noise noise = init.normal(torch.FloatTensor(num_data, 1), std=std) # print("noise tensor = {}".format(noise)) # x x = init.uniform(torch.Tensor(num_data, 1), a=-10, b=10) print("value of x ={}".format(x)) # y if True: y = -x**2 + noise y = y + noise else: y = noise print(type(x)) print(x.size())
def __init__(self, depth, pretrained=True, num_features=0, norm=False, dropout=0, num_classes=0): super(ResNet, self).__init__() self.depth = depth self.pretrained = pretrained # Construct base (pretrained) resnet if depth not in ResNet.__factory: raise KeyError("Unsupported depth:", depth) self.base = ResNet.__factory[depth](pretrained=pretrained) self.num_features = num_features self.norm = norm self.dropout = dropout self.has_embedding = num_features > 0 self.num_classes = num_classes out_planes = self.base.fc.in_features # Append new layers if self.has_embedding: self.layer2_maxpool = nn.MaxPool2d(kernel_size=4, stride=4) self.layer3_maxpool = nn.MaxPool2d(kernel_size=2, stride=2) self.l23_conv = nn.Conv2d(1024, 512, kernel_size=3, stride=2, padding=1) self.l43_conv = nn.Conv2d(2048, 1024, kernel_size=3, stride=2, padding=1) self.feat = nn.Linear(out_planes, self.num_features) self.feat_bn = nn.BatchNorm1d(self.num_features) init.kaiming_normal(self.l23_conv.weight, mode='fan_out') init.constant(self.l23_conv.bias, 0) init.kaiming_normal(self.l43_conv.weight, mode='fan_out') init.constant(self.l43_conv.bias, 0) init.kaiming_normal(self.feat.weight, mode='fan_out') init.constant(self.feat.bias, 0) init.constant(self.feat_bn.weight, 1) init.constant(self.feat_bn.bias, 0) else: # Change the num_features to CNN output channels self.num_features = out_planes if self.dropout > 0: self.drop = nn.Dropout(self.dropout) if self.num_classes > 0: self.classifier = nn.Linear(self.num_features, self.num_classes) init.normal(self.classifier.weight, std=0.001) init.constant(self.classifier.bias, 0) if not self.pretrained: self.reset_params()
def convert_to_rpp(self): self.pool = RPP(vector_length=2048, p=self.p) init.normal(self.pool.classifier.weight, std=self.rpp_std) init.constant(self.pool.classifier.bias, 0) return self
def init_conv_bn(conv_model, bn_model): # conv_model.weight.numel() # bn_model.bias.numel() init.xavier_normal(conv_model.weight.data, gain=0.02) init.normal(bn_model.weight.data, mean=1.0, std=0.02) init.constant(bn_model.bias.data, val=0.0)
from fpn import FPN50 from retinanet import RetinaNet print('Loading pretrained ResNet50 model..') d = torch.load('./model/resnet50.pth') print('Loading into FPN50..') fpn = FPN50() dd = fpn.state_dict() for k in d.keys(): if not k.startswith('fc'): # skip fc layers dd[k] = d[k] print('Saving RetinaNet..') net = RetinaNet() for m in net.modules(): if isinstance(m, nn.Conv2d): init.normal(m.weight, mean=0, std=0.01) if m.bias is not None: init.constant(m.bias, 0) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() pi = 0.01 init.constant(net.cls_head[-1].bias, -math.log((1 - pi) / pi)) net.fpn.load_state_dict(dd) torch.save(net.state_dict(), 'net.pth') print('Done!')
from retinanet import RetinaNet print('Loading pretrained ResNet50 model..') d = torch.load('./model/resnet50.pth') print('Loading into FPN50..') fpn = FPN50() dd = fpn.state_dict() for k in d.keys(): if not k.startswith('fc'): # skip fc layers dd[k] = d[k] print('Saving RetinaNet..') net = RetinaNet() for m in net.modules(): if isinstance(m, nn.Conv2d): init.normal(m.weight, mean=0, std=0.01) if m.bias is not None: init.constant(m.bias, 0) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_() pi = 0.01 init.constant(net.cls_head[-1].bias, -math.log((1-pi)/pi)) net.fpn.load_state_dict(dd) torch.save(net.state_dict(), 'net.pth') print('Done!')
def weights_init_bn(m): classname = m.__class__.__name__ # print(classname) if classname.find('BatchNorm2d') != -1: init.normal(m.weight.data, 1.0, 0.02) init.constant(m.bias.data, 0.0)