def initialize_weights(model, init): """Initialize the given model according to a chosen function """ for layer in model.modules(): if isinstance(layer, nn.Conv2d): init(layer.weight.data) elif isinstance(layer, nn.ConvTranspose2d): init(layer.weight.data) elif isinstance(layer, nn.BatchNorm2d): layer.weight.data.normal_(1.0, 0.2) layer.bias.data.zero_()
def linear_model(data, nh=50): (x_train, y_train, x_valid, y_valid) = data # weights = torch.randn(784, 10) # bias = torch.zeros(10) train_mean, train_std = x_train.mean(), x_train.std() x_train = normalize(x_train, train_mean, train_std) x_valid = normalize(x_valid, train_mean, train_std) nrows, ncols = x_train.shape nclasses = y_train.max() + 1 ## kaiming initialization #w1 = torch.randn(ncols, nh) / math.sqrt(2/ncols) w1 = torch.zeros(ncols, nh) w1 = init(w1, mode='fan_out') b1 = torch.zeros(nh) w2 = torch.randn(nh, 1) / math.sqrt(nh) b2 = torch.zeros(1) # forward output = model(x_valid, w1, b1, w2, b2) res = mse(output, y_valid.float()) y = bias + matmul(weights, data)
def Parameter(shape=None, init=xavier_uniform): if hasattr(init, 'shape'): assert not shape return nn.Parameter(torch.Tensor(init)) shape = (1, shape) if type(shape) == int else shape return nn.Parameter(init(torch.Tensor(*shape)))
def icnr(self, scale=2, init=nn.init.kaiming_normal_): ni, nf, h, w = self.conv0.weight.shape ni2 = int(ni / (scale**2)) k = init(torch.zeros([ni2, nf, h, w])).transpose(0, 1) k = k.contiguous().view(ni2, nf, -1) k = k.repeat(1, 1, scale**2) k = k.contiguous().view([nf, ni, h, w]).transpose(0, 1) self.conv0.weight.data = k
def __init__(self, input_dim, output_dim, filter_size, mask_type = None): self.input_dim = input_dim self.output_dim = output_dim self.filter_size = filter_size if mask_type is not None: self.mask = mask(filter_size, input_dim, output_dim, mask_type) self.weights = nn.Parameter(nn.init(torch.empty(output_dim, input_dim, filter_size, filter_size)))
def uniform_init(module, init_params={ "lower": 0, "upper": 1 }, types=[], category="all"): return init(module, types, uniform_, init_params, category)
def normal_init(module, init_params={ "mean": 0, "std": 1 }, types=[], category="all"): return init(module, types, normal_, init_params, category)
def sparsity_init(module, init_params={ "sparsity": 0.1, "std": 0.01 }, types=[], category="all"): return init(module, types, sparse_, init_params, category)
def Parameter(shape=None, init=xavier_uniform): """ initializes Tensor with the input shape (if present) """ if hasattr(init, 'shape'): assert not shape return nn.Parameter(torch.Tensor(init)) shape = (shape, 1) if type(shape) == int else shape return nn.Parameter(init(torch.Tensor(*shape)))
def kaiming_normal_init(module, init_params={ "a": 0, "mode": "fan_in", "nonlinearity": "leaky_relu" }, types=[], category="weight"): return init(module, types, kaiming_normal_, init_params, category)
def init_linear(m): return init(m, nn.init.orthogonal_, lambda x: nn.init.constant_(x, 0), nn.init.calculate_gain('relu'))
def orthorgonal_init(module, init_params={"gain": 1}, types=[], category="all"): return init(module, types, orthogonal_, init_params, category)
def constant_init(module, value, types=[], category="all"): return init(module, types, constant_, {"val": value}, category)
def ones_init(module, types=[], category="all"): return init(module, types, constant_, {"val": 1}, category)
def Parameter(shape=None, init=xavier_uniform): shape = (shape, 1) if type(shape) == int else shape return nn.Parameter(init(torch.Tensor(*shape)))
def zeros_init(module, types=[], category="all"): return init(module, types, constant_, {"val": 0}, category)
def eye_init(module, types=[], category="all"): return init(module, types, eye_, None, category)
def weight_init(m): if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear): nn.init.xavier_uniform(m.weight.data) nn.init(constant(m.bias, 0.1))
from torch.autograd import Function import torch.nn as nn import torch.nn.functional as F import torchvision import TRNmodule import math from colorama import init from colorama import Fore, Back, Style import pdb torch.manual_seed(1) torch.cuda.manual_seed_all(1) init(autoreset=True) # definition of Gradient Reversal Layer class GradReverse(Function): @staticmethod def forward(ctx, x, beta): ctx.beta = beta return x.view_as(x) @staticmethod def backward(ctx, grad_output): grad_input = grad_output.neg() * ctx.beta return grad_input, None # definition of Gradient Scaling Layer class GradScale(Function):
def xavier_uniform_init(module, init_params={"gain": 1}, types=[], category="weight"): return init(module, types, xavier_uniform_, init_params, category)
def dirac_init(module, types=[], category="all"): return init(module, types, dirac_, None, category)