def step(self, feed_dict, reduce_func=default_reduce_func):
assert self._model.training, 'Step a evaluation-mode model.'
self.trigger_event('step:before', self)
feed_dict = as_variable(feed_dict)
begin = time.time()
self.trigger_event('forward:before', self, feed_dict)
loss, monitors, output_dict = self._model(feed_dict)
self.trigger_event('forward:after', self, feed_dict, loss, monitors,
output_dict)
loss = reduce_func('loss', loss)
monitors = {k: reduce_func(k, v) for k, v in monitors.items()}
loss_f = as_float(loss)
monitors_f = as_float(monitors)
self._optimizer.zero_grad()
self.trigger_event('backward:before', self, loss)
loss.backward()
self.trigger_event('backward:after', self, loss)
self._optimizer.step()
end = time.time()
self.trigger_event('step:after', self)
return loss_f, monitors_f, output_dict, {'time/gpu': end - begin}
def test_deconv2d(self):
in_tensor = as_variable(torch.randn(16, 8, 4, 2))
self._test_conv_size(in_tensor,
jacnn.Deconv2dLayer(8, 16, 3, scale_factor=2),
(16, 16, 8, 4))
self._test_conv_size(
in_tensor,
jacnn.Deconv2dLayer(8, 16, 3, scale_factor=2,
algo='convtranspose'), (16, 16, 8, 4))
def validate_step(self, feed_dict, metric, meters=None):
feed_dict_np = as_numpy(feed_dict)
feed_dict = mark_volatile(as_variable(feed_dict))
output_dict = self._model(feed_dict)
output_dict_np = as_numpy(output_dict)
result = as_float(metric(feed_dict_np, output_dict_np))
if meters is not None:
meters.update(result)
return result
def __call__(self, ind):
raw_img, img, img_embedding, cap, cap_ext = self.dataset[ind]
img_embedding_precomp = self.model.img_enc(as_cuda(as_variable(img_embedding).unsqueeze(0)))
img = as_variable(img)
img.requires_grad = True
img_embedding_a = img_embedding = self.image_encoder(as_cuda(img.unsqueeze(0)))
img_embedding = self.model.img_enc(img_embedding)
txt = [cap]
txt.extend(cap_ext)
txt_embeddings, txt_var = self.enc_txt(txt)
return Record(
raw_img, cap, cap_ext,
img, img_embedding, img_embedding_precomp,
txt_var, txt_embeddings[0], txt_embeddings[1:]
)
def evaluate(self, feed_dict):
assert not self._model.training, 'Evaluating a training-mode model.'
begin = time.time()
feed_dict = as_variable(feed_dict)
feed_dict = mark_volatile(feed_dict)
output_dict = self._model(feed_dict)
end = time.time()
return output_dict, dict(gpu_time=end - begin)
def test_deconv2d(self):
in_tensor = as_variable(torch.randn(16, 8, 4, 2))
self._test_conv_size(
in_tensor, jacnn.ConvTranspose2d(8, 16, 3, padding_mode='same'),
(16, 16, 4, 2))
self._test_conv_size(in_tensor,
jacnn.ConvTranspose2d(8,
16,
3,
2,
padding_mode='same'),
(16, 16, 8, 4),
scale_factor=2)
def train_step(self, feed_dict, meters=None):
assert self._model.training
feed_dict = as_variable(feed_dict)
self._optimizer.zero_grad()
loss, monitors, output_dict = self._model(feed_dict)
loss.backward()
self._optimizer.step()
loss, monitors = map(as_float, [loss, monitors])
if meters is not None:
meters.update(loss=loss)
meters.update(monitors)
return as_float(loss)
def _prepare_batch(sents, projector):
if isinstance(sents, six.string_types):
sents = [sents]
sents = [np.array(projector(s)) for s in sents]
lengths = [len(s) for s in sents]
sents = _pad_sequences(sents, 0, max(lengths))
idx = np.array(sorted(range(len(lengths)), key=lambda x: lengths[x], reverse=True))
inv = np.array(sorted(range(len(lengths)), key=lambda x: idx[x]))
sents = sents[idx]
lengths = np.array(lengths)[idx].tolist()
sents = as_variable(sents)
if torch.cuda.is_available():
sents = sents.cuda()
return sents, lengths, idx, inv
def test_conv2d(self):
in_tensor = as_variable(torch.randn(16, 8, 4, 2))
self._test_conv_size(in_tensor, jacnn.Conv2d(8, 16, 3, padding=1),
(16, 16, 4, 2))
self._test_conv_size(
in_tensor, jacnn.Conv2d(8, 16, 2, stride=2, padding_mode='valid'),
(16, 16, 2, 1))
self._test_conv_size(in_tensor,
jacnn.Conv2d(8, 16, 3, padding_mode='same'),
(16, 16, 4, 2))
self._test_conv_size(
in_tensor,
jacnn.Conv2d(8,
16, (5, 3),
padding_mode='same',
border_mode='replicate'), (16, 16, 4, 2))
self._test_conv_size(
in_tensor,
jacnn.Conv2d(8,
16, (5, 3),
padding_mode='same',
border_mode='reflect'), (16, 16, 4, 2))
def _inference_model(self, feed_dict):
feed_dict = as_variable(feed_dict)
mark_volatile(feed_dict)
return as_numpy(self._model(feed_dict))
def test_resize_conv2d(self):
in_tensor = as_variable(torch.randn(16, 8, 4, 2))
self._test_conv_size(in_tensor,
jacnn.ResizeConv2d(8, 16, 3, scale_factor=2),
(16, 16, 8, 4))
def enc_txt(self, caps):
sents, lengths, _, inv = _prepare_batch(caps, self.projector)
inv = var_with(as_variable(inv), sents)
out, x = self.model.txt_enc.forward(sents, lengths, True)
return out[inv], x