def __init__(self, dataset_path: Path, ids: List[str], preload: bool):
self.dataset_path = Path(dataset_path)
self.features_path = (dataset_path / "features" /
"ICEP_V3_global_pool_skip_8_direct_resize")
self.cached_data = None
if preload:
self.cached_data = {}
for id_ in tqdm(ids, desc="preload videos"):
np_array = self.load_from_file(id_)
shared_array = utils.make_shared_array(np_array)
self.cached_data[id_] = shared_array
def __init__(self, dataset_path: Path, dataset_features: str,
ids: List[str], preload: bool):
self.dataset_path = Path(dataset_path)
self.h5_path = dataset_path / f"video_feat_{dataset_features}.h5"
self.cached_data = None
if preload:
self.cached_data = {}
h5file = h5py.File(self.h5_path, "r")
for id_ in tqdm(ids, desc="preload videos"):
np_array = h5file[id_]
shared_array = utils.make_shared_array(np_array)
self.cached_data[id_] = shared_array
def __init__(self,
dataset_path: Path,
ids: List[str],
metadata_name: str = "default",
preload=True):
self.h5_path = (dataset_path / f"text_{metadata_name}.h5")
lens_file = (dataset_path / f"text_lens_{metadata_name}.json")
self.lens = json.load(lens_file.open("rt", encoding="utf8"))
self.cached_data = None
if preload:
h5file = h5py.File(self.h5_path, "r")
self.cached_data = {}
for id_ in tqdm(ids, desc="preload text"):
np_array = h5file[id_]
shared_array = utils.make_shared_array(np_array)
self.cached_data[id_] = shared_array
h5file.close()
def __init__(self):
glove_path = Path("glove_vocab")
# vocab_path = glove_path / "activitynet_vocab.dill"
# self.vocab = dill.load(open(vocab_path, 'rb'))
# vocab = Vocab()
# vocab.word2idx = self.vocab.word2idx
# vocab.idx2word = self.vocab.idx2word
# vocab.idx = self.vocab.idx
# vocab.dump(glove_path / "vocab.json")
# exit()
self.vocab = Vocab()
self.vocab.load(glove_path / "vocab.json")
mapping_path = glove_path / "precomp_anet_w2v_total.npz"
npz_file = np.load(str(mapping_path))
np_arr = npz_file[npz_file.files[0]]
np_arr = np_arr.astype(np.float)
self.shared_array = utils.make_shared_array(np_arr)
assert np_arr.shape[0] == len(self.vocab)
self.feature_dim = 300