def read_data(self, path): if self.no_cache: data = indexed_dataset.IndexedDataset(path, fix_lua_indexing=True) else: data = indexed_dataset.IndexedCachedDataset(path, fix_lua_indexing=True) return data
def read_data(self, path): A_path = path + "-A" B_path = path + "-B" if self.no_cache: A_data = indexed_dataset.IndexedDataset(A_path, fix_lua_indexing=True) B_data = indexed_dataset.IndexedDataset(B_path, fix_lua_indexing=True) else: A_data = indexed_dataset.IndexedCachedDataset( A_path, fix_lua_indexing=True) B_data = indexed_dataset.IndexedCachedDataset( B_path, fix_lua_indexing=True) self.a_size = A_data.sizes self.b_size = B_data.sizes return (A_data, B_data)
def read_data(self, path): q_path = path + "-Q" a_path = path + "-A" if self.no_cache: q_data = indexed_dataset.IndexedDataset(q_path, fix_lua_indexing=True) a_data = indexed_dataset.IndexedDataset(a_path, fix_lua_indexing=True) else: q_data = indexed_dataset.IndexedCachedDataset( q_path, fix_lua_indexing=True) a_data = indexed_dataset.IndexedCachedDataset( a_path, fix_lua_indexing=True) self.q_size = q_data.sizes self.a_size = a_data.sizes return q_data, a_data