def test_poisson_nll_loss_reduction_modes(self): input = torch.tensor([0.5, 1.5, 2.5], device=device) target = torch.tensor([1., 2., 3.], device=device) component_wise_loss = torch.exp(input) - target * input self.assertEqual(component_wise_loss, F.poisson_nll_loss(input, target, reduction='none')) self.assertEqual(torch.sum(component_wise_loss), F.poisson_nll_loss(input, target, reduction='sum')) self.assertEqual(torch.mean(component_wise_loss), F.poisson_nll_loss(input, target, reduction='mean')) with self.assertRaisesRegex(ValueError, 'is not valid'): F.poisson_nll_loss(input, target, reduction='total')
def train_batch(self, x, optim): """ Trains on a data batch, with the given optimizer... """ optim.zero_grad() if self.use_reparam: output, mu, logvar = self(x) output += EPS loss = loss_function(output, x, mu, logvar) loss.backward() else: output = self(x) + EPS if self.loss == 'poisson': loss = F.poisson_nll_loss(output, x, log_input=False, full=True, reduction='sum') elif self.loss == 'l1': loss = F.l1_loss(output, x, reduction='sum') elif self.loss == 'mse': loss = F.mse_loss(output, x, reduction='sum') loss.backward() optim.step() self.clamp_m() return loss.item()
def forward(self): a = torch.randn(3, 2) b = torch.rand(3, 2) c = torch.rand(3) log_probs = torch.randn(50, 16, 20).log_softmax(2).detach() targets = torch.randint(1, 20, (16, 30), dtype=torch.long) input_lengths = torch.full((16, ), 50, dtype=torch.long) target_lengths = torch.randint(10, 30, (16, ), dtype=torch.long) return len( F.binary_cross_entropy(torch.sigmoid(a), b), F.binary_cross_entropy_with_logits(torch.sigmoid(a), b), F.poisson_nll_loss(a, b), F.cosine_embedding_loss(a, b, c), F.cross_entropy(a, b), F.ctc_loss(log_probs, targets, input_lengths, target_lengths), # F.gaussian_nll_loss(a, b, torch.ones(5, 1)), # ENTER is not supported in mobile module F.hinge_embedding_loss(a, b), F.kl_div(a, b), F.l1_loss(a, b), F.mse_loss(a, b), F.margin_ranking_loss(c, c, c), F.multilabel_margin_loss(self.x, self.y), F.multilabel_soft_margin_loss(self.x, self.y), F.multi_margin_loss(self.x, torch.tensor([3])), F.nll_loss(a, torch.tensor([1, 0, 1])), F.huber_loss(a, b), F.smooth_l1_loss(a, b), F.soft_margin_loss(a, b), F.triplet_margin_loss(a, b, -b), # F.triplet_margin_with_distance_loss(a, b, -b), # can't take variable number of arguments )
def train_batch(self, x, optim, batches=None): """ Trains on a data batch, with the given optimizer... """ optim.zero_grad() if self.use_reparam: output, mu, logvar = self.forward(x, batches) output += EPS loss = loss_function(output, x, mu, logvar) loss.backward() else: output, w = self.forward(x, batches) output += EPS if self.loss == 'poisson': loss = F.poisson_nll_loss(output, x, log_input=False, full=True, reduction='sum') elif self.loss == 'l1': loss = F.l1_loss(output, x, reduction='sum') elif self.loss == 'mse': loss = F.mse_loss(output, x, reduction='sum') if self.use_multibatch_loss: loss += self.multibatch_loss_weight * multibatch_loss( w, batches, self.num_batches) loss.backward() optim.step() self.clamp_m() return loss.item()
def reconstruction_error(self, input_tensor: torch.Tensor, hidden: torch.Tensor, dim: int) -> torch.Tensor: """ Compute the log probability of the original feature under p(x|z). Parameters ---------- input_tensor: torch.Tensor Original feature. hidden: torch.Tensor Latest decoder hidden state. dim: int Current feature dimension. Returns ------- reconstr_error: torch.Tensor Log probability of the input feature under the decoder's distribution. """ lambda_ = torch.exp(self.lambda_(hidden)).int().squeeze(1) err = F.poisson_nll_loss(input_tensor, lambda_, reduction="none", log_input=True) return err
def get_loss(loss_function, output, label, use_gpu): ''' get objective loss of model and backprograte to compute gradients some loss function not impelement ''' if not isinstance(loss_function, str): raise TypeError('loss_function should be str object') label = np.asarray(label) if loss_function == 'binary_cross_entropy': loss = F.binary_cross_entropy(output, label) elif loss_function == 'poisson_nll_loss': loss = F.poisson_nll_loss(output, label) elif loss_function == 'cross_entropy': loss = F.cross_entropy(output, label) elif loss_function == 'hinge_embedding_loss': loss = F.hinge_embedding_loss(output, label) elif loss_function == 'margin_ranking_loss': loss = F.margin_ranking_loss(output, label) elif loss_function == 'multilabel_soft_margin_loss': loss = F.multilabel_soft_margin_loss(output, label) elif loss_function == 'multi_margin_loss': loss = F.multi_margin_loss(output, label) elif loss_function == 'nll_loss': if use_gpu: label = Variable(torch.LongTensor(label).cuda()) label = Variable(torch.LongTensor(label)) loss = F.nll_loss(output, label) elif loss_function == 'binary_cross_entropy_with_logits': loss = F.binary_cross_entropy_with_logits(output, label) return loss
def loss(self, y_pred: Dict[str, torch.Tensor], target: torch.Tensor) -> torch.Tensor: return F.poisson_nll_loss(super().to_prediction(y_pred), target, log_input=True, full=False, eps=1e-6, reduction="none")
def loss_function(recon_x, x, mu, logvar, yhat, y): if loss_type=="binary": BCE = F.binary_cross_entropy(recon_x, x.view(-1, 68*68), reduction='sum') if loss_type=="poisson" BCE = F.poisson_nll_loss(recon_x , x, reduction='sum', log_input=True) NCE = F.mse_loss(yhat, y, reduction='sum') KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) return 0.01 * (BCE + KLD) + NCE
def closure(): if torch.is_grad_enabled(): # Clear cummulated gradients in every epoch optimizer.zero_grad() # Get output from the model outputs = self(x) # Compute average loss for grad loss = F.poisson_nll_loss(outputs, y) if loss.requires_grad: # Get gradients for parameters loss.backward() return loss
def loss_function(recon_x, x, mu, logvar): #BCE = poisson_loss(recon_x, x) BCE = F.poisson_nll_loss(recon_x, x, log_input=False, full=False, reduction='sum') #BCE = F.binary_cross_entropy(recon_x, x.view(-1, 784), reduction='sum') # see Appendix B from VAE paper: # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014 # https://arxiv.org/abs/1312.6114 # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2) KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) return BCE + KLD
def test_poisson_nll_loss(self): inp = torch.randn(32, 128, device='cuda', dtype=self.dtype, requires_grad=True) target = torch.randn(32, 128, device='cuda', dtype=self.dtype, requires_grad=False) output = F.poisson_nll_loss(inp, target, log_input=True, full=False, size_average=None, eps=1e-08, reduce=None, reduction='mean')
def configure_criterion(self, y, t): criterion = F.cross_entropy(y, t) if self.hparams.criterion == "cross_entropy": criterion = F.cross_entropy(y, t) elif self.hparams.criterion == "binary_cross_entropy": criterion = F.binary_cross_entropy(y, t) elif self.hparams.criterion == "binary_cross_entropy_with_logits": criterion = F.binary_cross_entropy_with_logits(y, t) elif self.hparams.criterion == "poisson_nll_loss": criterion = F.poisson_nll_loss(y, t) elif self.hparams.criterion == "hinge_embedding_loss": criterion = F.hinge_embedding_loss(y, t) elif self.hparams.criterion == "kl_div": criterion = F.kl_div(y, t) elif self.hparams.criterion == "l1_loss": criterion = F.l1_loss(y, t) elif self.hparams.criterion == "mse_loss": criterion = F.mse_loss(y, t) elif self.hparams.criterion == "margin_ranking_loss": criterion = F.margin_ranking_loss(y, t) elif self.hparams.criterion == "multilabel_margin_loss": criterion = F.multilabel_margin_loss(y, t) elif self.hparams.criterion == "multilabel_soft_margin_loss": criterion = F.multilabel_soft_margin_loss(y, t) elif self.hparams.criterion == "multi_margin_loss": criterion = F.multi_margin_loss(y, t) elif self.hparams.criterion == "nll_loss": criterion = F.nll_loss(y, t) elif self.hparams.criterion == "smooth_l1_loss": criterion = F.smooth_l1_loss(y, t) elif self.hparams.criterion == "soft_margin_loss": criterion = F.soft_margin_loss(y, t) return criterion
def fit(self, x, y, epochs, optim, lr): # Set the model into training mode self.train() num_cells = y.shape[1] device = y.device # Initialization min_loss = torch.ones(num_cells, dtype=torch.float32, device=device) * 1e9 best_weight = torch.ones_like(self.linear.weight.data) best_bias = torch.ones_like(self.linear.bias.data) if optim == 'adam': # Use Adam to fit the model optimizer = torch.optim.Adam(self.parameters(), lr=lr) else: # Use LBFGS to fit the model optimizer = torch.optim.LBFGS(self.parameters(), lr=lr) for epoch in tqdm(range(1, epochs + 1), 'Epoch: ', leave=False): if optim == 'adam': # Clear cummulated gradients in every epoch optimizer.zero_grad() # Get output from the model outputs = self(x) # Compute poisson negative log likelihood loss for each cell loss_cells = F.poisson_nll_loss(outputs, y, reduction='none') loss_cells = torch.sum(loss_cells, dim=0) # Compute average loss for grad loss = torch.sum(loss_cells) / num_cells # Get gradients for parameters loss.backward() # Update parameters optimizer.step() else: def closure(): if torch.is_grad_enabled(): # Clear cummulated gradients in every epoch optimizer.zero_grad() # Get output from the model outputs = self(x) # Compute average loss for grad loss = F.poisson_nll_loss(outputs, y) if loss.requires_grad: # Get gradients for parameters loss.backward() return loss # Update parameters optimizer.step(closure) # Get output from the model outputs = self(x) # Compute poisson negative log likelihood loss for each cell loss_cells = F.poisson_nll_loss(outputs, y, reduction='none') loss_cells = torch.sum(loss_cells, dim=0) loss = torch.sum(loss_cells) / num_cells # Update min loss, best weight/bias for each cell mask = loss_cells < min_loss min_loss[mask] = loss_cells[mask].detach( ) # Don't grad on this tensor best_weight[mask] = self.linear.weight.data[mask] best_bias[mask] = self.linear.bias.data[mask] # if epoch % 200 == 0: # print(f'Epoch: {epoch} Loss: {loss.item()}') # print(f'Training end, min loss: {torch.sum(min_loss)/num_cells}') return min_loss.cpu().numpy(), best_weight.cpu().numpy( ), best_bias.cpu().numpy()
def poisson_nll(y_pred, y_true): return F.poisson_nll_loss(y_pred, y_true)
def poisson_nll_loss(input, target, *args, **kwargs): return F.poisson_nll_loss(input.F, target, *args, **kwargs)
def poisson_nll_loss_cpu(y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor: y_pred = y_pred.detach().cpu() return func.poisson_nll_loss(y_pred, y_true)
def loss_function(recon_x, that, x, t , mu, logvar): # BCE = F.binary_cross_entropy(recon_x, x , reduction='sum') BCE = F.poisson_nll_loss(recon_x , x, reduction='sum', log_input=True) NCE = F.mse_loss(that, t, reduction='sum') KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp()) return 0.01*(BCE + KLD) + NCE
def adjusted_poisson_nll_loss_cpu(y_pred: torch.Tensor, y_true: torch.Tensor, a: torch.Tensor, b: torch.Tensor) -> torch.Tensor: y_pred = y_pred.detach().cpu() return func.poisson_nll_loss(y_pred - torch.log(a) + torch.log(b), y_true)