def predict(self, url, text):
with torch.no_grad():
detectron_features = self.get_detectron_features(url)
sample = Sample()
processed_text = self.text_processor({"text": text})
#sample.text = processed_text["text"]
sample.text_len = len(processed_text["tokens"])
encoded_input = tokenizer(text, return_tensors='pt')
sample.input_ids = encoded_input.input_ids
sample.input_mask = encoded_input.attention_mask
sample.segment_ids = encoded_input.token_type_ids
sample.image_feature_0 = detectron_features
sample.image_info_0 = Sample(
{"max_features": torch.tensor(100, dtype=torch.long)})
sample_list = SampleList([sample])
sample_list = sample_list.to("cuda")
output = self.visual_bert(sample_list)
gc.collect()
torch.cuda.empty_cache()
return output
def load_item(self, idx):
sample_info = self.imdb[idx]
current_sample = Sample()
processed_sentence = self.text_processor(
{"text": sample_info["sentence"]})
current_sample.text = processed_sentence["text"]
if "input_ids" in processed_sentence:
current_sample.update(processed_sentence)
if self._use_features is True:
# Remove sentence id from end
identifier = "-".join(sample_info["identifier"].split("-")[:-1])
# Load img0 and img1 features
sample_info["feature_path"] = "{}-img0.npy".format(identifier)
features = self.features_db[idx]
current_sample.img0 = Sample()
current_sample.img0.update(features)
sample_info["feature_path"] = "{}-img1.npy".format(identifier)
features = self.features_db[idx]
current_sample.img1 = Sample()
current_sample.img1.update(features)
is_correct = 1 if sample_info["label"] == "True" else 0
current_sample.targets = torch.tensor(is_correct, dtype=torch.long)
return current_sample
def _load_objects(self, idx):
image_info = self._get_image_info(idx)
image_height = image_info["height"]
image_width = image_info["width"]
object_map = {}
objects = []
for obj in image_info["objects"]:
obj["synsets"] = self.synset_processor({"tokens":
obj["synsets"]})["text"]
obj["names"] = self.name_processor({"tokens":
obj["names"]})["text"]
obj["height"] = obj["h"] / image_height
obj.pop("h")
obj["width"] = obj["w"] / image_width
obj.pop("w")
obj["y"] /= image_height
obj["x"] /= image_width
obj["attributes"] = self.attribute_processor(
{"tokens": obj["attributes"]})["text"]
obj = Sample(obj)
object_map[obj["object_id"]] = obj
objects.append(obj)
objects = SampleList(objects)
return objects, object_map
def load_item(self, idx):
sample_info = self.annotation_db[idx]
sample_info = self.preprocess_sample_info(sample_info)
current_sample = Sample()
if self._dataset_type != "test":
text_processor_argument = {"tokens": sample_info["caption_tokens"]}
processed_caption = self.text_processor(text_processor_argument)
current_sample.text = processed_caption["text"]
current_sample.caption_id = torch.tensor(sample_info["caption_id"],
dtype=torch.int)
current_sample.caption_len = torch.tensor(len(
sample_info["caption_tokens"]),
dtype=torch.int)
if isinstance(sample_info["image_id"], int):
current_sample.image_id = torch.tensor(sample_info["image_id"],
dtype=torch.int)
else:
current_sample.image_id = object_to_byte_tensor(
sample_info["image_id"])
if self._use_features is True:
features = self.features_db[idx]
current_sample.update(features)
# Add reference captions to sample
current_sample = self.add_reference_caption(sample_info,
current_sample)
return current_sample
def build_bbox_tensors(infos, max_length):
num_bbox = min(max_length, len(infos))
# After num_bbox, everything else should be zero
coord_tensor = torch.zeros((max_length, 4), dtype=torch.float)
width_tensor = torch.zeros(max_length, dtype=torch.float)
height_tensor = torch.zeros(max_length, dtype=torch.float)
bbox_types = ["xyxy"] * max_length
infos = infos[:num_bbox]
sample = Sample()
for idx, info in enumerate(infos):
bbox = info["bounding_box"]
x = bbox["top_left_x"]
y = bbox["top_left_y"]
width = bbox["width"]
height = bbox["height"]
coord_tensor[idx][0] = x
coord_tensor[idx][1] = y
coord_tensor[idx][2] = x + width
coord_tensor[idx][3] = y + height
width_tensor[idx] = width
height_tensor[idx] = height
sample.coordinates = coord_tensor
sample.width = width_tensor
sample.height = height_tensor
sample.bbox_types = bbox_types
return sample
def __getitem__(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
if self._use_features is True:
features = self.features_db[idx]
if hasattr(self, "transformer_bbox_processor"):
features["image_info_0"] = self.transformer_bbox_processor(
features["image_info_0"]
)
if self.config.get("use_image_feature_masks", False):
current_sample.update(
{
"image_labels": self.masked_region_processor(
features["image_feature_0"]
)
}
)
current_sample.update(features)
current_sample = self._add_masked_question(sample_info, current_sample)
return current_sample
def test_beam_search(self):
vocab = text_utils.VocabFromText(self.VOCAB_EXAMPLE_SENTENCES)
model_config = self.config.model_config.butd
model = TestDecoderModel(model_config, vocab)
model.build()
model.eval()
expected_tokens = {
1: [1.0, 23.0, 1.0, 24.0, 29.0, 37.0, 40.0, 17.0, 29.0, 2.0],
2: [1.0, 0.0, 8.0, 1.0, 28.0, 25.0, 2.0],
8: [1.0, 34.0, 1.0, 13.0, 1.0, 2.0],
16: [1.0, 25.0, 18.0, 2.0],
}
for batch_size in [1, 2, 8, 16]:
samples = []
for _ in range(batch_size):
sample = Sample()
sample.dataset_name = "coco"
sample.dataset_type = "test"
sample.image_feature_0 = torch.randn(100, 2048)
sample.answers = torch.zeros((5, 10), dtype=torch.long)
samples.append(sample)
sample_list = SampleList(samples)
tokens = model(sample_list)["captions"]
self.assertEqual(np.trim_zeros(tokens[0].tolist()),
expected_tokens[batch_size])
def __getitem__(self, idx: int) -> Sample:
sample = Sample()
sample[self.data_item_key] = torch.tensor(
idx, dtype=torch.float32).unsqueeze(-1)
if self.always_one:
sample.targets = torch.tensor(0, dtype=torch.long)
return sample
def __getitem__(self, idx):
record = self.video_meta[idx]
sampled_frame_indices = self._sample_indices(record)
video_path = record.path
video_start = record.start
video_en = record.end
video_level_label = record.label
image_tmpl,indices = self._get_img_file_temp(record,sampled_frame_indices)
ref_img_filename = reading_with_exception_handling(record, image_tmpl, int(indices[0]))
temp_img_ = self._load_image(ref_img_filename)[0]
ref_size = temp_img_.size
tracklet_info = self._parse_kmot_tracklet(record.rois)
filtered_boxes_with_track_by_indice = self._pick_mot_roi_by_indices(indices,tracklet_info,ref_size)
filtered_mot_box_dict = self._mot_list_to_indice_dict(indices,filtered_boxes_with_track_by_indice)
raw_gt_boxes_by_frames, box_labels_by_frames = self._get_gt_patch(indices,video_path,filtered_mot_box_dict)
processed_patch_boxes,targets = self._generate_patch_input(raw_gt_boxes_by_frames,box_labels_by_frames,ref_size,image_tmpl)
current_sample = Sample()
current_sample.targets = targets
current_sample.human_box = processed_patch_boxes
return current_sample
def __getitem__(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
text_processor_argument = {"text": sample_info["question_str"]}
processed_question = self.text_processor(text_processor_argument)
current_sample.text = processed_question["text"]
if "input_ids" in processed_question:
current_sample.update(processed_question)
current_sample.question_id = torch.tensor(
sample_info["question_id"], dtype=torch.int
)
if isinstance(sample_info["image_id"], int):
current_sample.image_id = torch.tensor(
sample_info["image_id"], dtype=torch.int
)
else:
current_sample.image_id = sample_info["image_id"]
if self._use_features is True:
features = self.features_db[idx]
if hasattr(self, "bbox_processor"):
features["image_info_0"] = self.bbox_processor(features["image_info_0"])
current_sample.update(features)
# Depending on whether we are using soft copy this can add
# dynamic answer space
current_sample = self.add_answer_info(sample_info, current_sample)
return current_sample
def load_item(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
if self._use_features:
features = self.features_db[idx]
if hasattr(self, "transformer_bbox_processor"):
features["image_info_0"] = self.transformer_bbox_processor(
features["image_info_0"]
)
if self.config.get("use_image_feature_masks", False):
current_sample.update(
{
"image_labels": self.masked_region_processor(
features["image_feature_0"]
)
}
)
current_sample.update(features)
else:
image_path = str(sample_info["image_name"]) + ".jpg"
current_sample.image = self.image_db.from_path(image_path)["images"][0]
current_sample = self._add_masked_question(sample_info, current_sample)
if self._add_answer:
current_sample = self.add_answer_info(sample_info, current_sample)
return current_sample
def __getitem__(self, idx):
if len(self.video_clips) == 0:
self.load_df()
video, audio, info = self.video_clips.get_clip(idx)
text = self.text_list[idx]
actual_idx = self.ids_list[idx]
label = [
self.class_to_idx[class_name] for class_name in self.labels[idx]
]
one_hot_label = torch.zeros(len(self.class_to_idx))
one_hot_label[label] = 1
if self.video_processor is not None:
video = self.video_processor(video)
if self.audio_processor is not None:
audio = self.audio_processor(audio)
sample = Sample()
sample.id = object_to_byte_tensor(actual_idx)
sample.video = video
sample.audio = audio
sample.update(self.text_processor({"text": text}))
sample.targets = one_hot_label
return sample
def __getitem__(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
#plot = sample_info["plotr"]
#if isinstance(plot, list):
# plot = plot[0]
raw_text = sample_info["name"]
processed_sentence = self.text_processor({"text": raw_text})
current_sample.text = processed_sentence["text"]
if "input_ids" in processed_sentence:
current_sample.update(processed_sentence)
"""
if self._use_features is True:
features = self.features_db[idx]
current_sample.update(features)
"""
current_sample.answers, current_sample.targets = self.labels(sample_info, idx)
#import pdb;pdb.set_trace()
print(idx)
current_sample.id = sample_info["idx"]
#import pdb;pdb.set_trace()
return current_sample
def load_item(self, idx):
sample_info = self.annotation_db[idx]
sample_info = self.preprocess_sample_info(sample_info)
current_sample = Sample()
if self._dataset_type != "test":
text_processor_argument = {"tokens": sample_info["caption_tokens"]}
processed_caption = self.text_processor(text_processor_argument)
current_sample.text = processed_caption["text"]
current_sample.caption_id = torch.tensor(sample_info["caption_id"],
dtype=torch.int)
current_sample.caption_len = torch.tensor(len(
sample_info["caption_tokens"]),
dtype=torch.int)
current_sample.image_id = object_to_byte_tensor(
sample_info["image_id"])
if self._use_features:
features = self.features_db[idx]
current_sample.update(features)
else:
image_path = str(sample_info["image_name"]) + ".jpg"
current_sample.image = self.image_db.from_path(
image_path)["images"][0]
# Add reference captions to sample
current_sample = self.add_reference_caption(sample_info,
current_sample)
return current_sample
def __getitem__(self, idx):
img, target = self.coco_dataset[idx]
image_id = self.coco_dataset.ids[idx]
target = {"image_id": image_id, "annotations": target}
img, target = self._load_coco_annotations(
img, target, load_attributes=self.config.load_attributes)
transform_out = self.detection_image_and_target_processor({
"img":
img,
"target":
target,
"dataset_type":
self._dataset_type
})
img = transform_out["img"]
target = transform_out["target"]
current_sample = Sample()
current_sample.image_id = torch.tensor(image_id, dtype=torch.long)
current_sample.image = img
current_sample.targets_enc = object_to_byte_tensor(
target, max_size=self.config.max_target_enc_size)
current_sample.orig_size = target["orig_size"].clone().detach()
return current_sample
def build_random_sample_list():
first = Sample()
first.x = random.randint(0, 100)
first.y = torch.rand((5, 4))
first.z = Sample()
first.z.x = random.randint(0, 100)
first.z.y = torch.rand((6, 4))
second = Sample()
second.x = random.randint(0, 100)
second.y = torch.rand((5, 4))
second.z = Sample()
second.z.x = random.randint(0, 100)
second.z.y = torch.rand((6, 4))
return SampleList([first, second])
def __getitem__(self, idx: int) -> Type[Sample]:
sample_info = self.annotation_db[idx]
current_sample = Sample()
if "question_tokens" in sample_info:
text_processor_argument = {
"tokens": sample_info["question_tokens"],
"text": sample_info["question_str"],
}
else:
text_processor_argument = {"text": sample_info["question"]}
processed_question = self.text_processor(text_processor_argument)
current_sample.update(processed_question)
current_sample.id = torch.tensor(int(sample_info["question_id"]),
dtype=torch.int)
if self._use_features:
features = self.features_db[idx]
if hasattr(self, "transformer_bbox_processor"):
features["image_info_0"] = self.transformer_bbox_processor(
features["image_info_0"])
current_sample.update(features)
else:
image_path = sample_info["image_name"] + ".jpg"
current_sample.image = self.image_db.from_path(
image_path)["images"][0]
current_sample = self.add_answer_info(sample_info, current_sample)
return current_sample
def test_nucleus_sampling(self):
vocab = text_utils.VocabFromText(self.VOCAB_EXAMPLE_SENTENCES)
model_config = self.config.model_config.butd
model = TestDecoderModel(model_config, vocab)
model.build()
model.eval()
sample = Sample()
sample.dataset_name = "coco"
sample.dataset_type = "test"
sample.image_feature_0 = torch.randn(100, 2048)
sample.answers = torch.zeros((5, 10), dtype=torch.long)
sample_list = SampleList([sample])
tokens = model(sample_list)["captions"]
# these are expected tokens for sum_threshold = 0.5
expected_tokens = [
1.0,
29.0,
11.0,
11.0,
39.0,
10.0,
31.0,
4.0,
19.0,
39.0,
2.0,
]
self.assertEqual(tokens[0].tolist(), expected_tokens)
def __getitem__(self, idx):
sample_info = self.annotation_db[idx]
sample_info = self.preprocess_sample_info(sample_info)
current_sample = Sample()
processed_text = self.text_processor({"text": sample_info["text"]})
current_sample.text = processed_text["text"]
if "input_ids" in processed_text:
current_sample.update(processed_text)
current_sample.id = torch.tensor(int(sample_info["id"]),
dtype=torch.int)
# Instead of using idx directly here, use sample_info to fetch
# the features as feature_path has been dynamically added
features = self.features_db.get(sample_info)
if hasattr(self, "transformer_bbox_processor"):
features["image_info_0"] = self.transformer_bbox_processor(
features["image_info_0"])
current_sample.update(features)
if "label" in sample_info:
current_sample.targets = torch.tensor(sample_info["label"],
dtype=torch.long)
return current_sample
def add_ocr_details(self, sample_info, sample):
if self.use_ocr:
# Preprocess OCR tokens
ocr_tokens = [
self.ocr_token_processor({"text": token})["text"]
for token in sample_info["ocr_tokens"]
]
# Get embeddings for tokens
context = self.context_processor({"tokens": ocr_tokens})
sample.context = context["text"]
sample.context_tokens = context["tokens"]
sample.context_feature_0 = context["text"]
sample.context_info_0 = Sample()
sample.context_info_0.max_features = context["length"]
order_vectors = torch.eye(len(sample.context_tokens))
order_vectors[context["length"] :] = 0
sample.order_vectors = order_vectors
if self.use_ocr_info and "ocr_info" in sample_info:
sample.ocr_bbox = self.bbox_processor({"info": sample_info["ocr_info"]})[
"bbox"
]
return sample
def __getitem__(self, idx: int) -> Sample:
sample_info = self.annotation_db[idx]
current_sample = Sample()
processed_caption = self.masked_token_processor({
"text_a":
sample_info["caption"],
"text_b":
"",
"is_correct":
True
})
current_sample.update(processed_caption)
current_sample.image_id = sample_info["image_id"]
current_sample.feature_path = sample_info["feature_path"]
# Get the image features
if self._use_features:
features = self.features_db[idx]
image_info_0 = features["image_info_0"]
if image_info_0 and "image_id" in image_info_0.keys():
image_info_0["feature_path"] = image_info_0["image_id"]
image_info_0.pop("image_id")
current_sample.update(features)
return current_sample
def __call__(self, image_tensor, text_input=None):
'''
Allow model to receive both multi-inputs and single image-inputs // Bojia Mao
'''
text = self.processor_dict["text_processor"]({"text": self.text})
sample = Sample()
if text_input == None:
sample.text = text["text"]
else:
self.__text = text_input
sample.text = text_input
if "input_ids" in text:
sample.update(text)
sample.image = image_tensor
sample_list = SampleList([sample])
sample_list = sample_list.to(
torch.device("cuda:0" if torch.cuda.is_available() else "cpu"))
output = self.model(sample_list)
scores = nn.functional.softmax(output["scores"], dim=1)
return scores
def __getitem__(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
text_processor_argument = {"text": sample_info["question_str"]}
processed_question = self.text_processor(text_processor_argument)
current_sample.text = processed_question["text"]
current_sample.text_mask = processed_question["text_mask"]
current_sample.raw_question = sample_info["question_str"]
if "input_ids" in processed_question:
current_sample.update(processed_question)
current_sample.question_id = torch.tensor(sample_info["question_id"],
dtype=torch.int)
if isinstance(sample_info["image_id"], int):
current_sample.image_id = torch.tensor(sample_info["image_id"],
dtype=torch.int)
else:
current_sample.image_id = sample_info["image_id"]
if self._use_features is True:
idx = int(self.img_info[str(sample_info["image_id"])]['index'])
current_sample.img_feature = torch.from_numpy(self.img[idx])
# Depending on whether we are using soft copy this can add
# dynamic answer space
current_sample = self.add_answer_info(sample_info, current_sample)
return current_sample
def test_forward(self):
model_config = self.config.model_config.cnn_lstm
cnn_lstm = CNNLSTM(model_config)
cnn_lstm.build()
cnn_lstm.init_losses()
self.assertTrue(isinstance(cnn_lstm, torch.nn.Module))
test_sample = Sample()
test_sample.text = torch.randint(1, 79, (10, ), dtype=torch.long)
test_sample.image = torch.randn(3, 320, 480)
test_sample.targets = torch.randn(32)
test_sample_list = SampleList([test_sample])
test_sample_list.dataset_type = "train"
test_sample_list.dataset_name = "clevr"
test_sample_list = test_sample_list.to(get_current_device())
cnn_lstm = cnn_lstm.to(get_current_device())
output = cnn_lstm(test_sample_list)
scores = output["scores"]
loss = output["losses"]["train/clevr/logit_bce"]
np.testing.assert_almost_equal(loss.item(), 19.2635, decimal=4)
self.assertEqual(scores.size(), torch.Size((1, 32)))
def load_item(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
processed_sentence = self.text_processor(
{"text": sample_info["sentence2"]})
current_sample.text = processed_sentence["text"]
if "input_ids" in processed_sentence:
current_sample.update(processed_sentence)
if self._use_features is True:
# Remove sentence id from end
identifier = sample_info["Flikr30kID"].split(".")[0]
# Load img0 and img1 features
sample_info["feature_path"] = "{}.npy".format(identifier)
features = self.features_db[idx]
if hasattr(self, "bbox_processor"):
features["image_info_0"] = self.bbox_processor(
features["image_info_0"])
current_sample.update(features)
label = LABEL_TO_INT_MAPPING[sample_info["gold_label"]]
current_sample.targets = torch.tensor(label, dtype=torch.long)
return current_sample
def __getitem__(self, idx: int) -> Sample:
sample_info = self.annotation_db[idx]
current_sample = Sample()
processed_caption = self.masked_token_processor({
"text_a":
sample_info["caption"],
"text_b":
"",
"is_correct":
True
})
current_sample.update(processed_caption)
current_sample.image_id = sample_info["image_id"]
current_sample.feature_path = sample_info["feature_path"]
# Get the image features
if self._use_features:
features = self.features_db[idx]
image_info_0 = features["image_info_0"]
if image_info_0 and "image_id" in image_info_0.keys():
image_info_0["feature_path"] = image_info_0["image_id"]
image_info_0.pop("image_id")
current_sample.update(features)
elif self._use_images:
image_id = sample_info["image_id"]
dataset = sample_info["dataset_id"]
if "mscoco" in dataset:
image_id = image_id.rjust(12, "0")
assert (len(self.image_db.from_path(image_id)["images"]) !=
0), f"image id: {image_id} not found"
current_sample.image = self.image_db.from_path(
image_id)["images"][0]
return current_sample
def load_item(self, idx):
sample_info = self.imdb[idx]
current_sample = Sample()
if self._use_features is True:
features = self.features_db[idx]
image_labels = []
for i in range(features["image_feature_0"].shape[0]):
prob = random.random()
# mask token with 15% probability
if prob < 0.15:
prob /= 0.15
if prob < 0.9:
features["image_feature_0"][i] = 0
image_labels.append(1)
else:
# no masking token (will be ignored by loss function later)
image_labels.append(-1)
item = {}
if self.config.get("use_image_feature_masks", False):
item["image_labels"] = image_labels
current_sample.update(item)
current_sample.update(features)
current_sample = self._add_masked_caption(sample_info, current_sample)
return current_sample
def __getitem__(self, idx):
sample_info = self.annotation_db[idx]
sample_info = self.preprocess_sample_info(sample_info)
current_sample = Sample()
# breaking change from VQA2Dataset: load question_id
current_sample.question_id = torch.tensor(
sample_info["question_id"], dtype=torch.int
)
if isinstance(sample_info["image_id"], int):
current_sample.image_id = str(sample_info["image_id"])
else:
current_sample.image_id = sample_info["image_id"]
if self._use_features is True:
features = self.features_db[idx]
current_sample.update(features)
current_sample = self.add_sample_details(sample_info, current_sample)
current_sample = self.add_answer_info(sample_info, current_sample)
# only the 'max_features' key is needed
# pop other keys to minimize data loading overhead
if hasattr(current_sample, "image_info_0"):
for k in list(current_sample.image_info_0):
if k != "max_features":
current_sample.image_info_0.pop(k)
if hasattr(current_sample, "image_info_1"):
for k in list(current_sample.image_info_1):
if k != "max_features":
current_sample.image_info_1.pop(k)
return current_sample
def load_item(self, idx):
sample_info = self.annotation_db[idx]
current_sample = Sample()
# if "question_tokens" in sample_info:
if (("question_tokens" in sample_info) and ("question_str" in sample_info)):
text_processor_argument = {
"tokens": sample_info["question_tokens"],
"text": sample_info["question_str"],
}
else:
text_processor_argument = {"text": sample_info["question"]}
processed_question = self.text_processor(text_processor_argument)
current_sample.text = processed_question["text"]
if "input_ids" in processed_question:
current_sample.update(processed_question)
current_sample.question_id = torch.tensor(
sample_info["question_id"], dtype=torch.int
)
if isinstance(sample_info["image_id"], int):
current_sample.image_id = torch.tensor(
sample_info["image_id"], dtype=torch.int
)
else:
current_sample.image_id = sample_info["image_id"]
if (("question_tokens" in sample_info) and ("question_str" in sample_info)):
current_sample.text_len = torch.tensor(
len(sample_info["question_tokens"]), dtype=torch.int
)
if self._use_features:
features = self.features_db[idx]
if self.max_features:
features["image_info_0"]["num_boxes"] = self.max_features
features["image_info_0"]["max_features"] = self.max_features
features["image_info_0"]["bbox"] = features["image_info_0"]["bbox"][0:self.max_features, :]
features["image_feature_0"] = features["image_feature_0"][0:self.max_features, :]
if hasattr(self, "transformer_bbox_processor"):
features["image_info_0"] = self.transformer_bbox_processor(
features["image_info_0"]
)
current_sample.update(features)
else:
image_path = sample_info["image_name"] + ".jpg"
current_sample.image = self.image_db.from_path(image_path)["images"][0]
# Add details for OCR like OCR bbox, vectors, tokens here
current_sample = self.add_ocr_details(sample_info, current_sample)
# Depending on whether we are using soft copy this can add
# dynamic answer space
current_sample = self.add_answer_info(sample_info, current_sample)
return current_sample
def _test_retrieval_recall_at_k_metric(self, metric, value):
sample = Sample()
predicted = dict()
torch.manual_seed(1234)
predicted["targets"] = torch.rand((10, 4))
predicted["scores"] = torch.rand((10, 4))
self.assertAlmostEqual(float(metric.calculate(sample, predicted)),
value)