def forward(self, input): # if activation bias is used on leaky relu else, directly in the linear function if self.activation: out = F.linear(input, self.weight * self.scale) out = fused_leaky_relu(out, self.bias * self.lr_mul) else: out = F.linear(input, self.weight * self.scale, bias=self.bias * self.lr_mul) return out
def _in_proj(self, input, start=0, end=None): weight = self.in_proj_weight bias = self.in_proj_bias weight = weight[start:end, :] if bias is not None: bias = bias[start:end] return F.linear(input, weight, bias)
def forward(self, input, sample=False, calculate_log_probs=False): # 1. Sample weights and bias from variational posterior if self.training or sample: weight = self.weight.sample() bias = self.bias.sample() else: weight = self.weight.mu bias = self.bias.mu # 2. Update log_prior and log_posterior according to current approximation if self.training or calculate_log_probs: self.log_prior = self.weight_prior.log_prob( weight) + self.bias_prior.log_prob(bias) self.log_variational_posterior = self.weight.log_prob( weight) + self.bias.log_prob(bias) else: self.log_prior, self.log_variational_posterior = 0, 0 # 3. Do a forward pass through the layer return F.linear(input, weight, bias)
def forward(self, input): return F.linear(input, self.mask * self.weight, self.bias)
def forward(self, input): if DEBUG: print("masked linear: ", torch.any(torch.isnan(input)), input.mean()) return F.linear(input, self.mask * self.weight, self.bias)