def test_perplexity(self):
nll = NLLLoss()
ppl = Perplexity()
nll.eval_batch(self.outputs, self.batch)
ppl.eval_batch(self.outputs, self.batch)
nll_loss = nll.get_loss()
ppl_loss = ppl.get_loss()
self.assertAlmostEqual(ppl_loss, math.exp(nll_loss))
def test_nllloss_WITH_OUT_SIZE_AVERAGE(self):
loss = NLLLoss(reduction='sum')
pytorch_loss = 0
pytorch_criterion = torch.nn.NLLLoss(reduction='sum')
for output, target in zip(self.outputs, self.targets):
loss.eval_batch(output, target)
pytorch_loss += pytorch_criterion(output, target)
loss_val = loss.get_loss()
self.assertAlmostEqual(loss_val, pytorch_loss.item())
def test_nllloss_WITH_OUT_SIZE_AVERAGE(self):
num_repeat = 10
loss = NLLLoss(reduction='sum')
pytorch_loss = 0
pytorch_criterion = torch.nn.NLLLoss(reduction='sum')
for _ in range(num_repeat):
for step, output in enumerate(self.outputs):
pytorch_loss += pytorch_criterion(output,
self.targets[:, step + 1])
loss.eval_batch(self.outputs, self.batch)
loss_val = loss.get_loss()
self.assertAlmostEqual(loss_val, pytorch_loss.item())
def test_nllloss(self):
num_batch = 10
loss = NLLLoss()
pytorch_loss = 0
pytorch_criterion = torch.nn.NLLLoss()
for _ in range(num_batch):
for step, output in enumerate(self.outputs):
pytorch_loss += pytorch_criterion(output,
self.targets[:, step + 1])
loss.eval_batch(self.outputs, self.batch)
loss_val = loss.get_loss()
pytorch_loss /= (num_batch * len(self.outputs))
self.assertAlmostEqual(loss_val, pytorch_loss.item())
def __init__(self,
vocab_size: int,
embedding_size: int,
n_hidden: int,
sos_token: int = 0,
eos_token: int = 1,
mask_token: int = 2,
max_output_length: int = 100,
rnn_cell: str = 'lstm') -> None:
self.decoder = DecoderRNN(vocab_size,
max_output_length,
embedding_size,
n_layers=n_hidden,
rnn_cell=rnn_cell,
use_attention=False,
bidirectional=False,
eos_id=eos_token,
sos_id=sos_token)
if torch.cuda.is_available(): self.decoder.cuda()
self.rnn_cell = rnn_cell
self.n_hidden = n_hidden
self.embedding_size = embedding_size
self.SOS_token = sos_token
self.EOS_token = eos_token
self.mask_token = mask_token
self.max_output_length = max_output_length
token_weights = torch.ones(vocab_size)
if torch.cuda.is_available(): token_weights = token_weights.cuda()
self.loss = NLLLoss(weight=token_weights, mask=mask_token)
self.optimizer = None
def __init__(self,
loss=NLLLoss(),
explosion_rate=120,
batch_size=1024,
polyglot=False):
self.loss = loss
self.batch_size = batch_size
self.polyglot = polyglot
self.explosion_rate = explosion_rate
def __init__(self,
expt_dir='experiment',
loss=[NLLLoss()],
loss_weights=None,
metrics=[],
batch_size=64,
eval_batch_size=128,
random_seed=None,
checkpoint_every=100,
print_every=100,
early_stopper=None,
anneal_middropout=0,
min_middropout=0.01):
self._trainer = "Simple Trainer"
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
k = NLLLoss()
self.loss = loss
self.metrics = metrics
self.loss_weights = loss_weights or len(loss) * [1.]
self.evaluator = Evaluator(loss=self.loss,
metrics=self.metrics,
batch_size=eval_batch_size)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.anneal_middropout = anneal_middropout
self.min_middropout = 0 if self.anneal_middropout == 0 else min_middropout
self.early_stopper = early_stopper
if early_stopper is not None:
assert self.early_stopper.mode == "min", "Can currently only be used with the loss, please use mode='min'"
if not os.path.isabs(expt_dir):
expt_dir = os.path.join(os.getcwd(), expt_dir)
self.expt_dir = expt_dir
if not os.path.exists(self.expt_dir):
os.makedirs(self.expt_dir)
self.batch_size = batch_size
self.logger = logging.getLogger(__name__)
def test_perplexity(self):
num_class = 5
num_batch = 10
batch_size = 5
outputs = [F.softmax(Variable(torch.randn(batch_size, num_class)))
for _ in range(num_batch)]
targets = [Variable(torch.LongTensor([random.randint(0, num_class - 1)
for _ in range(batch_size)]))
for _ in range(num_batch)]
nll = NLLLoss()
ppl = Perplexity()
for output, target in zip(outputs, targets):
nll.eval_batch(output, target)
ppl.eval_batch(output, target)
nll_loss = nll.get_loss()
ppl_loss = ppl.get_loss()
self.assertAlmostEqual(ppl_loss, math.exp(nll_loss))
def test_nllloss(self):
num_class = 5
num_batch = 10
batch_size = 5
outputs = [F.softmax(Variable(torch.randn(batch_size, num_class)))
for _ in range(num_batch)]
targets = [Variable(torch.LongTensor([random.randint(0, num_class - 1)
for _ in range(batch_size)]))
for _ in range(num_batch)]
loss = NLLLoss()
pytorch_loss = 0
pytorch_criterion = torch.nn.NLLLoss()
for output, target in zip(outputs, targets):
loss.eval_batch(output, target)
pytorch_loss += pytorch_criterion(output, target)
loss_val = loss.get_loss()
pytorch_loss /= num_batch
self.assertAlmostEqual(loss_val, pytorch_loss.data[0])
def __init__(self,
model_dir='experiment',
best_model_dir='experiment/best',
loss=NLLLoss(),
batch_size=64,
random_seed=None,
checkpoint_every=100,
print_every=100,
max_epochs=5,
max_steps=10000,
max_checkpoints_num=5,
best_ppl=100000.0,
device=None):
self._trainer = "Simple Trainer"
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.loss = loss
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.max_steps = max_steps
self.max_epochs = max_epochs
self.batch_size = batch_size
self.best_ppl = best_ppl
self.max_checkpoints_num = max_checkpoints_num
self.device = device
self.evaluator = Evaluator(loss=self.loss,
batch_size=batch_size,
device=device)
if not os.path.isabs(model_dir):
model_dir = os.path.join(os.getcwd(), model_dir)
self.model_dir = model_dir
if not os.path.exists(self.model_dir):
os.makedirs(self.model_dir)
if not os.path.isabs(best_model_dir):
best_model_dir = os.path.join(os.getcwd(), best_model_dir)
self.best_model_dir = best_model_dir
if not os.path.exists(self.best_model_dir):
os.makedirs(self.best_model_dir)
self.model_checkpoints = []
self.best_model_checkpoints = []
self.logger = logging.getLogger(__name__)
def __init__(self, experiment_directory='./experiment', loss=None, batch_size=64,
random_seed=None, checkpoint_every=100, print_every=100):
if loss is None:
loss = NLLLoss()
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.loss = loss
self.evaluator = Evaluator(loss=self.loss, batch_size=batch_size)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
self.batch_size = batch_size
self.experiment_directory = experiment_directory
if not os.path.exists(self.experiment_directory):
os.makedirs(self.experiment_directory)
