def save_assignments(self, prefix, mode="train"):
assignments = np.vstack(self.assignments_list)
qst_ids = []
for qid in self.qst_ids_list:
qst_ids.extend(qid)
print("shape of assignments: ", assignments.shape)
if mode == "train":
origin_qst_ids = self.origin_train_qst_ids
else:
origin_qst_ids = self.origin_test_qst_ids
assignments, qst_ids = cmf.reorder_assignments_using_qst_ids(
origin_qst_ids, qst_ids, assignments, is_subset=True)
# setting directory for saving assignments
save_dir = os.path.join(self.config["misc"]["result_dir"],
"assignments", mode)
io_utils.check_and_create_dir(save_dir)
# save assignments
save_hdf5_path = os.path.join(save_dir, prefix + "_assignment.h5")
hdf5_file = io_utils.open_hdf5(save_hdf5_path, "w")
hdf5_file.create_dataset("assignments",
dtype="int32",
data=assignments)
print("Assignments are saved in {}".format(save_hdf5_path))
save_json_path = os.path.join(save_dir, prefix + "_assignment.json")
for qsp in self.assignment_qst_ans_pairs:
for k, v in qsp.items():
qsp[k] = list(qsp[k])
io_utils.write_json(save_json_path, self.assignment_qst_ans_pairs)
print("Assignments (ans-qst) are saved in {}".format(save_json_path))
# save assignments of qst_ids
save_json_path = os.path.join(save_dir, prefix + "_qst_ids.json")
out = {}
out["question_ids"] = qst_ids
io_utils.write_json(save_json_path, out)
print("Saving is done: {}".format(save_json_path))
def generate_labels(self, config):
""" Generate and save labels for temporal language grouding
1)query_info (.json) with
- wtoi: word to index dictionary (vocabulary)
- itow: index to word dictionary (vocabulary)
- query_lengths: lengths for queries
2)query_labels (.h5): qid -> label
3)grounding_labels (.h5): qid -> label
"""
""" Query information """
if not os.path.exists(self.paths["query_labels"]):
# build vocabulary from training data
train_ann_path = "data/charades/annotations/charades_sta_train.txt"
train_aux_path = "data/charades/annotations/Charades_v1_train.csv"
train_anns, _, _ = self._load_annotation(train_ann_path,
train_aux_path)
wtoi = self._build_vocab(train_anns)
itow = {v: k for k, v in wtoi.items()}
# encode query and save labels (+lenghts)
L = config.get("max_length", 20)
encoded = self._encode_query(self.anns, wtoi, L)
query_labels = io_utils.open_hdf5(self.paths["query_labels"], "w")
for qid in tqdm(encoded["query_lengths"].keys(),
desc="Saving query"):
_ = query_labels.create_dataset(
str(qid), data=encoded["query_labels"][qid])
query_labels.close()
# save vocabulary and query length
query_info = {
"wtoi": wtoi,
"itow": itow,
"query_lengths": encoded["query_lengths"],
}
io_utils.write_json(self.paths["query_info"], query_info)
""" Grounding information """
if not os.path.exists(self.paths["grounding_info"]):
grd_dataset = io_utils.open_hdf5(self.paths["grounding_info"], "w")
start_pos = grd_dataset.create_group("start_pos")
end_pos = grd_dataset.create_group("end_pos")
att_masks = grd_dataset.create_group("att_mask")
for qid, ann in tqdm(self.anns.items(), desc="Gen. Grd. Labels"):
# get starting/ending positions
ts = ann["timestamps"]
vid_d = ann["duration"]
start = ts[0] / vid_d
end = ts[1] / vid_d
# get attention calibration mask
vid = ann["video_id"]
if self.feature_type == "I3D":
nfeats = np.load(self.feat_path.format(vid)).shape[0]
else:
raise NotImplementedError()
nfeats = min(nfeats, self.S)
fs = utils.timestamp_to_featstamp(ts, nfeats, vid_d)
att_mask = np.zeros((self.S))
att_mask[fs[0]:fs[1] + 1] = 1
_ = start_pos.create_dataset(qid, data=start, dtype="float")
_ = end_pos.create_dataset(qid, data=end, dtype="float")
_ = att_masks.create_dataset(qid, data=att_mask, dtype="float")
# save the encoded proposal labels and video ids
grd_dataset.close()