def reset_parameters(self): std = math.sqrt(2.0 / float(self.input_dim + self.output_dim)) a = math.sqrt( 3.0) * std # Calculate uniform bounds from standard deviation init._no_grad_uniform_(self.W.weight, -a, a) if self.W.bias is not None: init._no_grad_zero_(self.W.bias)
def xavier_uniform_(tensor, gain=1.): # type: (Tensor, float) -> Tensor r"""Fills the input `Tensor` with values according to the method described in `Understanding the difficulty of training deep feedforward neural networks` - Glorot, X. & Bengio, Y. (2010), using a uniform distribution. The resulting tensor will have values sampled from :math:`\mathcal{U}(-a, a)` where .. math:: a = \text{gain} \times \sqrt{\frac{6}{\text{fan\_in} + \text{fan\_out}}} Also known as Glorot initialization. Args: tensor: an n-dimensional `torch.Tensor` gain: an optional scaling factor Examples: >>> w = torch.empty(3, 5) >>> nn.init.xavier_uniform_(w, gain=nn.init.calculate_gain('relu')) """ fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) std = gain * math.sqrt(6.0 / float(fan_in + fan_out)) # a = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation a = std res = _no_grad_uniform_(tensor, -a, a) # print(res) return res
def xavier_uniform_small_init_(tensor, gain=1.): # type: (Tensor, float) -> Tensor fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) std = gain * math.sqrt(2.0 / float(fan_in + 4 * fan_out)) a = math.sqrt( 3.0) * std # Calculate uniform bounds from standard deviation return _no_grad_uniform_(tensor, -a, a)
def reset_parameters(self): fan_in = sum(self.input_dims) std = math.sqrt(2.0 / float(fan_in + self.output_dim)) a = math.sqrt( 3.0) * std # Calculate uniform bounds from standard deviation for transform in self.transforms: if self.init_type == "xavier": init._no_grad_uniform_(transform.weight, -a, a) else: print("did not implement he init") exit() init.zeros_(self.full_bias) for layer_norm in self.layer_norms: layer_norm.reset_parameters()
def __init__(self, n_inputs, n_outputs, synaptic_filter=filters.raised_cosine_pillow_08, n_basis_feedforward=1, n_basis_feedback=1, tau_ff=1, tau_fb=1, mu=0.5, device='cpu'): super(SNNLayer, self).__init__() self.device = device self.n_inputs = n_inputs self.n_outputs = n_outputs ### Feedforward connections self.n_basis_feedforward = n_basis_feedforward self.feedforward_filter = synaptic_filter( tau_ff, self.n_basis_feedforward, mu).transpose(0, 1).to(self.device) self.feedforward_filter.requires_grad = False self.tau_ff = tau_ff ### Feedback connections self.n_basis_feedback = n_basis_feedback self.feedback_filter = synaptic_filter(tau_fb, self.n_basis_feedback, mu).transpose(0, 1).to(self.device) self.feedback_filter.requires_grad = False self.tau_fb = tau_fb self.ff_weights = torch.nn.parameter.Parameter( torch.Tensor(n_outputs, n_inputs, n_basis_feedforward)) self.fb_weights = torch.nn.parameter.Parameter( torch.Tensor(n_outputs, n_basis_feedback)) self.bias = torch.nn.parameter.Parameter(torch.Tensor(n_outputs)) a = self.get_xavier() _no_grad_uniform_(self.ff_weights, -a, a) _no_grad_uniform_(self.fb_weights, -a, a) _no_grad_uniform_(self.bias, -a, a) self.spiking_history = torch.zeros([self.n_outputs, 2], requires_grad=False).to(self.device) self.potential = None ### Number of timesteps to keep in synaptic memory self.memory_length = max(self.tau_ff, self.tau_fb)
def enhanceUniform(tensor, dim, baseInitMethod, baseInitMethodParams): #Calcukate the bound if baseInitMethod == 'kaiming': bound = getUniformKaimingBound(tensor, **baseInitMethodParams) elif baseInitMethod == 'xavier': bound = getUniformXavierBound(tensor, **baseInitMethodParams) else: raise UnsupportedInitMethod('enhanceUniform.'+ str(baseInitMethod) +' unsupported method. Use \'kaiming\' or \'xavier\'') if dim == 0: #Regular case. This means we initialize the entire tensor from same normal distribution like baseInitMethod (both ) torchInit._no_grad_uniform_(tensor, -bound, bound) return for filt in tensor.data: #If dim is 1, then we initialize each filter if dim == 1: #According to formula of inversing uniform distribution. unif_1 = uniform.rvs(loc=0,scale = 1, size=1)[0] enhanceBound = (bound*unif_1)**2 enhanceBound = (enhanceBound*3)**0.5 torchInit._no_grad_uniform_(filt, -enhanceBound, enhanceBound) #Continue to next filter continue #If we got here, dim is 2, and initialize for each sub-filter #Calculate the bounds. According to formula of inversing uniform distribution. unif_1 = uniform.rvs(loc=0,scale = 1, size=filt.shape[0]) enhanceBound = (bound*unif_1)**2 enhanceBound = (enhanceBound*3)**0.5 for subfiltIdx, subfilt in enumerate(filt): torchInit._no_grad_uniform_(subfilt, -enhanceBound[subfiltIdx], enhanceBound[subfiltIdx]) return