def test_perplexity(self): nll = NLLLoss() ppl = Perplexity() nll.eval_batch(self.outputs, self.batch) ppl.eval_batch(self.outputs, self.batch) nll_loss = nll.get_loss() ppl_loss = ppl.get_loss() self.assertAlmostEqual(ppl_loss, math.exp(nll_loss))
def test_nllloss_WITH_OUT_SIZE_AVERAGE(self): loss = NLLLoss(reduction='sum') pytorch_loss = 0 pytorch_criterion = torch.nn.NLLLoss(reduction='sum') for output, target in zip(self.outputs, self.targets): loss.eval_batch(output, target) pytorch_loss += pytorch_criterion(output, target) loss_val = loss.get_loss() self.assertAlmostEqual(loss_val, pytorch_loss.item())
def test_nllloss_WITH_OUT_SIZE_AVERAGE(self): num_repeat = 10 loss = NLLLoss(reduction='sum') pytorch_loss = 0 pytorch_criterion = torch.nn.NLLLoss(reduction='sum') for _ in range(num_repeat): for step, output in enumerate(self.outputs): pytorch_loss += pytorch_criterion(output, self.targets[:, step + 1]) loss.eval_batch(self.outputs, self.batch) loss_val = loss.get_loss() self.assertAlmostEqual(loss_val, pytorch_loss.item())
def test_nllloss(self): num_batch = 10 loss = NLLLoss() pytorch_loss = 0 pytorch_criterion = torch.nn.NLLLoss() for _ in range(num_batch): for step, output in enumerate(self.outputs): pytorch_loss += pytorch_criterion(output, self.targets[:, step + 1]) loss.eval_batch(self.outputs, self.batch) loss_val = loss.get_loss() pytorch_loss /= (num_batch * len(self.outputs)) self.assertAlmostEqual(loss_val, pytorch_loss.item())
def __init__(self, vocab_size: int, embedding_size: int, n_hidden: int, sos_token: int = 0, eos_token: int = 1, mask_token: int = 2, max_output_length: int = 100, rnn_cell: str = 'lstm') -> None: self.decoder = DecoderRNN(vocab_size, max_output_length, embedding_size, n_layers=n_hidden, rnn_cell=rnn_cell, use_attention=False, bidirectional=False, eos_id=eos_token, sos_id=sos_token) if torch.cuda.is_available(): self.decoder.cuda() self.rnn_cell = rnn_cell self.n_hidden = n_hidden self.embedding_size = embedding_size self.SOS_token = sos_token self.EOS_token = eos_token self.mask_token = mask_token self.max_output_length = max_output_length token_weights = torch.ones(vocab_size) if torch.cuda.is_available(): token_weights = token_weights.cuda() self.loss = NLLLoss(weight=token_weights, mask=mask_token) self.optimizer = None
def __init__(self, loss=NLLLoss(), explosion_rate=120, batch_size=1024, polyglot=False): self.loss = loss self.batch_size = batch_size self.polyglot = polyglot self.explosion_rate = explosion_rate
def __init__(self, expt_dir='experiment', loss=[NLLLoss()], loss_weights=None, metrics=[], batch_size=64, eval_batch_size=128, random_seed=None, checkpoint_every=100, print_every=100, early_stopper=None, anneal_middropout=0, min_middropout=0.01): self._trainer = "Simple Trainer" self.random_seed = random_seed if random_seed is not None: random.seed(random_seed) torch.manual_seed(random_seed) k = NLLLoss() self.loss = loss self.metrics = metrics self.loss_weights = loss_weights or len(loss) * [1.] self.evaluator = Evaluator(loss=self.loss, metrics=self.metrics, batch_size=eval_batch_size) self.optimizer = None self.checkpoint_every = checkpoint_every self.print_every = print_every self.anneal_middropout = anneal_middropout self.min_middropout = 0 if self.anneal_middropout == 0 else min_middropout self.early_stopper = early_stopper if early_stopper is not None: assert self.early_stopper.mode == "min", "Can currently only be used with the loss, please use mode='min'" if not os.path.isabs(expt_dir): expt_dir = os.path.join(os.getcwd(), expt_dir) self.expt_dir = expt_dir if not os.path.exists(self.expt_dir): os.makedirs(self.expt_dir) self.batch_size = batch_size self.logger = logging.getLogger(__name__)
def test_perplexity(self): num_class = 5 num_batch = 10 batch_size = 5 outputs = [F.softmax(Variable(torch.randn(batch_size, num_class))) for _ in range(num_batch)] targets = [Variable(torch.LongTensor([random.randint(0, num_class - 1) for _ in range(batch_size)])) for _ in range(num_batch)] nll = NLLLoss() ppl = Perplexity() for output, target in zip(outputs, targets): nll.eval_batch(output, target) ppl.eval_batch(output, target) nll_loss = nll.get_loss() ppl_loss = ppl.get_loss() self.assertAlmostEqual(ppl_loss, math.exp(nll_loss))
def test_nllloss(self): num_class = 5 num_batch = 10 batch_size = 5 outputs = [F.softmax(Variable(torch.randn(batch_size, num_class))) for _ in range(num_batch)] targets = [Variable(torch.LongTensor([random.randint(0, num_class - 1) for _ in range(batch_size)])) for _ in range(num_batch)] loss = NLLLoss() pytorch_loss = 0 pytorch_criterion = torch.nn.NLLLoss() for output, target in zip(outputs, targets): loss.eval_batch(output, target) pytorch_loss += pytorch_criterion(output, target) loss_val = loss.get_loss() pytorch_loss /= num_batch self.assertAlmostEqual(loss_val, pytorch_loss.data[0])
def __init__(self, model_dir='experiment', best_model_dir='experiment/best', loss=NLLLoss(), batch_size=64, random_seed=None, checkpoint_every=100, print_every=100, max_epochs=5, max_steps=10000, max_checkpoints_num=5, best_ppl=100000.0, device=None): self._trainer = "Simple Trainer" self.random_seed = random_seed if random_seed is not None: random.seed(random_seed) torch.manual_seed(random_seed) self.loss = loss self.optimizer = None self.checkpoint_every = checkpoint_every self.print_every = print_every self.max_steps = max_steps self.max_epochs = max_epochs self.batch_size = batch_size self.best_ppl = best_ppl self.max_checkpoints_num = max_checkpoints_num self.device = device self.evaluator = Evaluator(loss=self.loss, batch_size=batch_size, device=device) if not os.path.isabs(model_dir): model_dir = os.path.join(os.getcwd(), model_dir) self.model_dir = model_dir if not os.path.exists(self.model_dir): os.makedirs(self.model_dir) if not os.path.isabs(best_model_dir): best_model_dir = os.path.join(os.getcwd(), best_model_dir) self.best_model_dir = best_model_dir if not os.path.exists(self.best_model_dir): os.makedirs(self.best_model_dir) self.model_checkpoints = [] self.best_model_checkpoints = [] self.logger = logging.getLogger(__name__)
def __init__(self, experiment_directory='./experiment', loss=None, batch_size=64, random_seed=None, checkpoint_every=100, print_every=100): if loss is None: loss = NLLLoss() if random_seed is not None: random.seed(random_seed) torch.manual_seed(random_seed) self.loss = loss self.evaluator = Evaluator(loss=self.loss, batch_size=batch_size) self.optimizer = None self.checkpoint_every = checkpoint_every self.print_every = print_every self.batch_size = batch_size self.experiment_directory = experiment_directory if not os.path.exists(self.experiment_directory): os.makedirs(self.experiment_directory)