def predict(self, url, feat_name, get_features=False):
with torch.no_grad():
detectron_features = get_detectron_features([url],
self.detection_model,
False, feat_name,
self.cuda_device)
# returns a single-element list
detectron_features = detectron_features[0]
sample = Sample()
sample.dataset_name = "coco"
sample.dataset_type = "test"
sample.image_feature_0 = detectron_features
sample.answers = torch.zeros((5, 10), dtype=torch.long)
sample_list = SampleList([sample])
sample_list = sample_list.to(self.cuda_device)
tokens = self.caption_model(sample_list)["captions"]
gc.collect()
torch.cuda.empty_cache()
if not get_features:
return tokens
else:
return tokens, detectron_features
def forward(self, images, image_scales, transitions=None):
feature_list = self.encoder(images, image_scales)
image_features = feature_list[0]
assert len(
feature_list) == 1, 'current model only support batch size 1'
sample = Sample()
sample.dataset_name = "coco"
sample.dataset_type = "test"
sample.image_feature_0 = image_features
# it seems answers work as a place holder here
# hence, it does not matter what it's size is
sample.answers = torch.zeros((1, 10), dtype=torch.long)
sample_list = SampleList([sample])
sample_list = sample_list.to(device)
# set_trace()
if transitions is not None:
sample_list.transitions = transitions
output = self.decoder(sample_list)
tokens = output['captions']
caption = tokens.tolist()[0]
caption = self.decoder.caption_processor(caption)['caption']
return caption
def test_nucleus_sampling(self):
vocab = text_utils.VocabFromText(self.VOCAB_EXAMPLE_SENTENCES)
model_config = self.config.model_attributes.butd
model = TestDecoderModel(model_config, vocab)
model.build()
model.to("cuda")
model.eval()
sample = Sample()
sample.dataset_name = "coco"
sample.dataset_type = "test"
sample.image_feature_0 = torch.randn(100, 2048)
sample.answers = torch.zeros((5, 10), dtype=torch.long)
sample_list = SampleList([sample])
tokens = model(sample_list)["captions"]
# these are expected tokens for sum_threshold = 0.5
expected_tokens = [
1.0000e+00, 2.9140e+03, 5.9210e+03, 2.2040e+03, 5.0550e+03,
9.2240e+03, 4.5120e+03, 1.8200e+02, 3.6490e+03, 6.4090e+03,
2.0000e+00
]
self.assertEqual(tokens[0].tolist(), expected_tokens)
def predict(self, url):
with torch.no_grad():
detectron_features = self.get_detectron_features(url)
sample = Sample()
sample.dataset_name = "coco"
sample.dataset_type = "test"
sample.image_feature_0 = detectron_features
sample.answers = torch.zeros((5, 10), dtype=torch.long)
sample_list = SampleList([sample])
sample_list = sample_list.to("cuda")
tokens = self.pythia_model(sample_list)["captions"]
gc.collect()
torch.cuda.empty_cache()
return tokens
