def initialize_weights(model, init):
"""Initialize the given model according to a chosen
function
"""
for layer in model.modules():
if isinstance(layer, nn.Conv2d):
init(layer.weight.data)
elif isinstance(layer, nn.ConvTranspose2d):
init(layer.weight.data)
elif isinstance(layer, nn.BatchNorm2d):
layer.weight.data.normal_(1.0, 0.2)
layer.bias.data.zero_()
def linear_model(data, nh=50):
(x_train, y_train, x_valid, y_valid) = data
# weights = torch.randn(784, 10)
# bias = torch.zeros(10)
train_mean, train_std = x_train.mean(), x_train.std()
x_train = normalize(x_train, train_mean, train_std)
x_valid = normalize(x_valid, train_mean, train_std)
nrows, ncols = x_train.shape
nclasses = y_train.max() + 1
## kaiming initialization
#w1 = torch.randn(ncols, nh) / math.sqrt(2/ncols)
w1 = torch.zeros(ncols, nh)
w1 = init(w1, mode='fan_out')
b1 = torch.zeros(nh)
w2 = torch.randn(nh, 1) / math.sqrt(nh)
b2 = torch.zeros(1)
# forward
output = model(x_valid, w1, b1, w2, b2)
res = mse(output, y_valid.float())
y = bias + matmul(weights, data)
def Parameter(shape=None, init=xavier_uniform):
if hasattr(init, 'shape'):
assert not shape
return nn.Parameter(torch.Tensor(init))
shape = (1, shape) if type(shape) == int else shape
return nn.Parameter(init(torch.Tensor(*shape)))
def icnr(self, scale=2, init=nn.init.kaiming_normal_):
ni, nf, h, w = self.conv0.weight.shape
ni2 = int(ni / (scale**2))
k = init(torch.zeros([ni2, nf, h, w])).transpose(0, 1)
k = k.contiguous().view(ni2, nf, -1)
k = k.repeat(1, 1, scale**2)
k = k.contiguous().view([nf, ni, h, w]).transpose(0, 1)
self.conv0.weight.data = k
def __init__(self, input_dim, output_dim, filter_size, mask_type = None):
self.input_dim = input_dim
self.output_dim = output_dim
self.filter_size = filter_size
if mask_type is not None:
self.mask = mask(filter_size, input_dim, output_dim, mask_type)
self.weights = nn.Parameter(nn.init(torch.empty(output_dim, input_dim, filter_size, filter_size)))
def uniform_init(module,
init_params={
"lower": 0,
"upper": 1
},
types=[],
category="all"):
return init(module, types, uniform_, init_params, category)
def normal_init(module,
init_params={
"mean": 0,
"std": 1
},
types=[],
category="all"):
return init(module, types, normal_, init_params, category)
def sparsity_init(module,
init_params={
"sparsity": 0.1,
"std": 0.01
},
types=[],
category="all"):
return init(module, types, sparse_, init_params, category)
def Parameter(shape=None, init=xavier_uniform):
"""
initializes Tensor with the input shape (if present)
"""
if hasattr(init, 'shape'):
assert not shape
return nn.Parameter(torch.Tensor(init))
shape = (shape, 1) if type(shape) == int else shape
return nn.Parameter(init(torch.Tensor(*shape)))
def kaiming_normal_init(module,
init_params={
"a": 0,
"mode": "fan_in",
"nonlinearity": "leaky_relu"
},
types=[],
category="weight"):
return init(module, types, kaiming_normal_, init_params, category)
def init_linear(m):
return init(m, nn.init.orthogonal_, lambda x: nn.init.constant_(x, 0),
nn.init.calculate_gain('relu'))
def orthorgonal_init(module,
init_params={"gain": 1},
types=[],
category="all"):
return init(module, types, orthogonal_, init_params, category)
def constant_init(module, value, types=[], category="all"):
return init(module, types, constant_, {"val": value}, category)
def ones_init(module, types=[], category="all"):
return init(module, types, constant_, {"val": 1}, category)
def Parameter(shape=None, init=xavier_uniform):
shape = (shape, 1) if type(shape) == int else shape
return nn.Parameter(init(torch.Tensor(*shape)))
def zeros_init(module, types=[], category="all"):
return init(module, types, constant_, {"val": 0}, category)
def eye_init(module, types=[], category="all"):
return init(module, types, eye_, None, category)
def weight_init(m):
if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):
nn.init.xavier_uniform(m.weight.data)
nn.init(constant(m.bias, 0.1))
from torch.autograd import Function import torch.nn as nn import torch.nn.functional as F import torchvision import TRNmodule import math from colorama import init from colorama import Fore, Back, Style import pdb torch.manual_seed(1) torch.cuda.manual_seed_all(1) init(autoreset=True) # definition of Gradient Reversal Layer class GradReverse(Function): @staticmethod def forward(ctx, x, beta): ctx.beta = beta return x.view_as(x) @staticmethod def backward(ctx, grad_output): grad_input = grad_output.neg() * ctx.beta return grad_input, None # definition of Gradient Scaling Layer class GradScale(Function):
def xavier_uniform_init(module,
init_params={"gain": 1},
types=[],
category="weight"):
return init(module, types, xavier_uniform_, init_params, category)
def dirac_init(module, types=[], category="all"):
return init(module, types, dirac_, None, category)
