def test_lu_weight_decomposition(self):
# ----------------------------------------------------------------------
# Prepare some dummy data
# ----------------------------------------------------------------------
data = torch.randn(2, 16, 32, 32)
weight_mat = 2 * torch.eye(16, 16)
# ----------------------------------------------------------------------
# Prepare the layer
# ----------------------------------------------------------------------
inv_conv = Invertible1x1Conv(16, lu_decomposition=True)
# initialize weights to get W = I
init.eye_(inv_conv.permutation)
init.eye_(inv_conv.lower)
init.zeros_(inv_conv.upper)
init.constant_(inv_conv.log_s, 0.6931471805599453)
init.constant_(inv_conv.sign_s, 1)
# ----------------------------------------------------------------------
# Assess the results are as expected
# ----------------------------------------------------------------------
out, log_det = inv_conv(data)
target_log_det = \
(torch.logdet(weight_mat) * data.size(2) * data.size(3)).expand(2)
# --------------
error_weight = torch.mean(torch.abs(out - 2 * data)).item()
error_log_det = torch.mean(torch.abs(log_det - target_log_det)).item()
self.assertLessEqual(error_weight, 1e-12)
self.assertLessEqual(error_log_det, 1e-12)
def __init__(self):
super(Point_Transform_Net, self).__init__()
self.k = 3
self.bn1 = nn.BatchNorm2d(64)
self.bn2 = nn.BatchNorm2d(128)
self.bn3 = nn.BatchNorm1d(1024)
self.conv1 = nn.Sequential(nn.Conv2d(6, 64, kernel_size=1, bias=False),
self.bn1,
nn.LeakyReLU(negative_slope=0.2))
self.conv2 = nn.Sequential(nn.Conv2d(64, 128, kernel_size=1, bias=False),
self.bn2,
nn.LeakyReLU(negative_slope=0.2))
self.conv3 = nn.Sequential(nn.Conv1d(128, 1024, kernel_size=1, bias=False),
self.bn3,
nn.LeakyReLU(negative_slope=0.2))
self.linear1 = nn.Linear(1024, 512, bias=False)
self.bn3 = nn.BatchNorm1d(512)
self.linear2 = nn.Linear(512, 256, bias=False)
self.bn4 = nn.BatchNorm1d(256)
self.transform = nn.Linear(256, 3*3)
init.constant_(self.transform.weight, 0)
init.eye_(self.transform.bias.view(3, 3))
def __init__(self, k): #, args):
super(Transform_Net, self).__init__()
#self.args = args
self.k = k
self.bn1 = nn.GroupNorm(32, 64)
self.bn1_2 = nn.GroupNorm(32, 64)
self.bn2 = nn.GroupNorm(32, 128)
self.bn3 = nn.GroupNorm(32, 1024)
self.conv1 = nn.Sequential(
nn.Conv2d(self.k * 2, 64, kernel_size=1, bias=False), self.bn1,
nn.LeakyReLU(negative_slope=0.2))
self.conv1_2 = nn.Sequential(
nn.Conv2d(64, 64, kernel_size=1, bias=False), self.bn1_2,
nn.LeakyReLU(negative_slope=0.2))
self.conv2 = nn.Sequential(
nn.Conv2d(64, 128, kernel_size=1, bias=False), self.bn2,
nn.LeakyReLU(negative_slope=0.2))
self.conv3 = nn.Sequential(
nn.Conv1d(128, 1024, kernel_size=1, bias=False), self.bn3,
nn.LeakyReLU(negative_slope=0.2))
self.linear1 = nn.Linear(1024, 512, bias=False)
self.bn3 = nn.GroupNorm(32, 512)
self.linear2 = nn.Linear(512, 256, bias=False)
self.bn4 = nn.GroupNorm(32, 256)
self.transform = nn.Linear(256, k * k) #3*3)
init.constant_(self.transform.weight, 0)
init.eye_(self.transform.bias.view(k, k)) #3, 3))
def __init__(self, args=None, normals=False):
super(Transform_Net, self).__init__()
self.args = args
self.k = 3
if normals:
self.initialFeatSize = 6
self.outputSize = 3
else:
self.initialFeatSize = 2 * 128
self.outputSize = 128
self.bn1 = nn.BatchNorm2d(64)
self.bn2 = nn.BatchNorm2d(128)
self.bn3 = nn.BatchNorm1d(1024)
self.conv1 = nn.Sequential(
nn.Conv2d(self.initialFeatSize, 64, kernel_size=1, bias=False),
self.bn1, nn.LeakyReLU(negative_slope=0.2))
self.conv2 = nn.Sequential(
nn.Conv2d(64, 128, kernel_size=1, bias=False), self.bn2,
nn.LeakyReLU(negative_slope=0.2))
self.conv3 = nn.Sequential(
nn.Conv1d(128, 1024, kernel_size=1, bias=False), self.bn3,
nn.LeakyReLU(negative_slope=0.2))
self.linear1 = nn.Linear(1024, 512, bias=False)
self.bn3 = nn.BatchNorm1d(512)
self.linear2 = nn.Linear(512, 256, bias=False)
self.bn4 = nn.BatchNorm1d(256)
self.transform = nn.Linear(256, self.outputSize * self.outputSize)
init.constant_(self.transform.weight, 0)
init.eye_(self.transform.bias.view(self.outputSize, self.outputSize))
def init_func(m):
classname = m.__class__.__name__
if classname.find('BatchNorm2d') != -1:
if hasattr(m, 'weight') and m.weight is not None:
init.normal_(m.weight.data, 1.0, gain)
if hasattr(m, 'bias') and m.bias is not None:
init.constant_(m.bias.data, 0.0)
elif hasattr(m, 'weight') and \
(classname.find('Conv') != -1 or \
classname.find('Linear') != -1):
if init_type == 'normal':
init.normal_(m.weight.data, 0.0, gain)
elif init_type == 'xavier':
init.xavier_normal_(m.weight.data, gain=gain)
elif init_type == 'xavier_uniform':
init.xavier_uniform_(m.weight.data, gain=1.0)
elif init_type == 'kaiming':
init.kaiming_normal_(m.weight.data, a=0, mode='fan_in')
elif init_type == 'orthogonal':
init.orthogonal_(m.weight.data, gain=gain)
elif init_type == 'identity':
init.eye_(m.weight.data)
elif init_type == 'none': # uses pytorch's default init method
m.reset_parameters()
else:
raise NotImplementedError('[%s] is not implemented' %
init_type)
if hasattr(m, 'bias') and m.bias is not None:
init.constant_(m.bias.data, 0.0)
def reset_parameters(self):
if self.temp_warp is not None:
init._no_grad_fill_(self.temp, 5.0)
init._no_grad_fill_(self.weight, -1.0)
with torch.no_grad():
self.weight.fill_diagonal_(1.0)
else:
init.eye_(self.weight)
def __init__(self, cf, confidence=None):
super(Baseline, self).__init__()
self.cf = cf
self.classifier = nn.Linear(cf + 1, cf + 1)
self.dummy = counting.Counter(cf,
already_sigmoided=True,
confidence=confidence)
init.eye_(self.classifier.weight)
def weights_init(m):
classname = m.__class__.__name__
if isinstance(m, nn.Linear):
init.xavier_normal_(m.weight, gain=np.sqrt(2.0))
elif classname.find('Conv') != -1:
init.xavier_normal_(m.weight, gain=np.sqrt(2.0))
elif classname.find('Linear') != -1:
init.eye_(m.weight)
elif classname.find('Emb') != -1:
init.normal(m.weight, mean=0, std=0.01)
def weights_init(m):
classname = m.__class__.__name__
if classname.find('Conv') != -1:
init.xavier_uniform_(m.weight.data)
elif classname.find('Gdn2d') != -1:
init.eye_(m.gamma.data)
init.constant_(m.beta.data, 1e-4)
elif classname.find('Gdn1d') != -1:
init.eye_(m.gamma.data)
init.constant_(m.beta.data, 1e-4)
def ME_weights_init_connections_identity(m):
classname = m.__class__.__name__
if isinstance(m, ME.MinkowskiConvolution):
m.kernel.fill_(1) # for 1*1 convolution is equivalent to identity
if m.bias is not None:
m.bias.data.fill_(0)
elif isinstance(m, ME.MinkowskiLinear):
eye_(m.linear.weight.data)
if m.linear.bias is not None:
m.linear.bias.data.fill_(0)
elif isinstance(m, ME.MinkowskiBatchNorm):
m.bn.reset_parameters()
def weights_init_connections_identity(m):
"""
For HOLMES: initialize identity connections
"""
classname = m.__class__.__name__
if classname.find('Conv') != -1:
m.weight.data.fill_(1) # for 1*1 convolution is equivalent to identity
if m.bias is not None:
m.bias.data.fill_(0)
elif classname.find('Linear') != -1:
eye_(m.weight.data)
if m.bias is not None:
m.bias.data.fill_(0)
def __init__(self):
super(FeatureTransformNet, self).__init__()
self.conv_block_1 = conv_bn_block(64, 64, 1)
self.conv_block_2 = conv_bn_block(64, 128, 1)
self.conv_block_3 = conv_bn_block(128, 1024, 1)
self.fc_block_4 = fc_bn_block(1024, 512)
self.fc_block_5 = fc_bn_block(512, 256)
self.transform = nn.Linear(256, 64*64)
# 转换矩阵初始化
# transform层的weight为[256 * 9]，bias为9；weight全部初始化为0， bias初始为[1, 0, 0, 0, 1, 0, 0, 0, 1]
init.constant_(self.transform.weight, 0)
init.eye_(self.transform.bias.view(64, 64))
def init_iDelta(tensor, delta=0.01):
'''initialize the tensor with I + \delta * random '''
r = torch.empty(tensor.shape)
i = init.eye_(tensor)
# fan_in, fan_out = init._calculate_fan_in_and_fan_out(tensor)
# a = math.sqrt(6/(fan_in+fan_out))# adding this scaling factor doesn't seem very different. same fast.
# return torch.mul(a, i) + torch.mul(delta, init.xavier_uniform(r))
return i + torch.mul(delta, init.uniform_(r, 0, 1.0))
def __init__(self):
super(InputTransformNet, self).__init__()
self.relu = nn.ReLU()
self.conv1 = nn.Conv1d(3, 64, 1)
self.bn1 = nn.BatchNorm1d(64)
self.conv2 = nn.Conv1d(64, 128, 1)
self.bn2 = nn.BatchNorm1d(128)
self.conv3 = nn.Conv1d(128, 1024, 1)
self.bn3 = nn.BatchNorm1d(1024)
self.fc1 = nn.Linear(1024, 512)
self.bn4 = nn.BatchNorm1d(512)
self.fc2 = nn.Linear(512, 256)
self.bn5 = nn.BatchNorm1d(256)
self.transform = nn.Linear(256, 9)
init.constant_(self.transform.weight, 0)
init.eye_(self.transform.bias.view(3, 3))
def reset_parameters(self):
"""
Initialize parameters following the way proposed in the paper.
"""
# The input-to-hidden weight matrix is initialized orthogonally.
init.orthogonal(self.weight_ih.data)
# The hidden-to-hidden weight matrix is initialized as an identity
# matrix.
eye_(self.weight_hh)
# The bias is just set to zero vectors.
init.constant(self.bias.data, val=0)
# Initialization of BN parameters.
self.bn_ih.reset_parameters()
self.bn_hh.reset_parameters()
self.bn_c.reset_parameters()
self.bn_ih.bias.data.fill_(0)
self.bn_hh.bias.data.fill_(0)
self.bn_ih.weight.data.fill_(0.1)
self.bn_hh.weight.data.fill_(0.1)
self.bn_c.weight.data.fill_(0.1)
def reset_prameters(self):
eye_(self.conv0.weight)
self.conv0.weight.requires_grad = False
eye_(self.conv0.lin.weight)
self.conv1.reset_parameters()
self.conv2.reset_parameters()
self.conv3.reset_parameters()
self.norm1.reset_parameters()
self.norm2.reset_parameters()
self.norm3.reset_parameters()
self.lin_GNN.reset_parameters()
self.out_conv1.reset_parameters()
self.out_conv2.reset_parameters()
self.out_conv3.reset_parameters()
self.out_norm1.reset_parameters()
self.out_norm2.reset_parameters()
self.out_norm3.reset_parameters()
self.lin_out.reset_parameters()
def init_weights(self):
init.eye_(self.fc1.weight)
def reset_parameters(self):
init.orthogonal_(self.weight_ih.data)
init.eye_(self.weight_hh)
init.constant_(self.bias.data, val=0)
init.eye_(self.trans)
# normal t2 = torch.Tensor(3, 1, 2) init.normal_(t2, mean=0, std=1) # constant t3 = torch.tensor([3, 1], dtype=torch.int64) init.constant_(t3, 2) t4 = torch.Tensor(3, 1, 2) # dim init.constant_(t4, 2) t5 = torch.tensor(np.ndarray([3, 1], dtype=np.float), dtype=torch.int64) init.constant_(t5, 2) # eye - 2 dim t6 = torch.Tensor(3, 3) # t6 = torch.Tensor(3, 3, 3) # error init.eye_(t6) # ones/zeros t7 = torch.empty(3, 1) # print(t7.dtype) init.ones_(t7) print(t7) # dirac # xavier_normal # kaiming_uniform # kaiming_normal
def __init__(self, cf):
super(Baseline, self).__init__()
self.cf = cf
self.classifier = nn.Linear(cf + 1, cf + 1)
self.dummy = counting.Counter(cf)
init.eye_(self.classifier.weight)
def __init__(self, cf):
super(Net, self).__init__()
self.cf = cf
self.counter = counting.Counter(cf)
self.classifier = nn.Linear(cf + 1, cf + 1)
init.eye_(self.classifier.weight)
def __init__(self, cf):
super(Net, self).__init__()
self.cf = cf
self.ctr = VQACntrModule(cf, already_sigmoided=True)
self.cfr = nn.Linear(cf + 1, cf + 1)
init.eye_(self.cfr.weight)
def init_parameter(self, parameter):
init.eye_(parameter)
def reset_parameters(self):
init.eye_(self.g)
