def read_data(self, path):
if self.no_cache:
data = indexed_dataset.IndexedDataset(path, fix_lua_indexing=True)
else:
data = indexed_dataset.IndexedCachedDataset(path,
fix_lua_indexing=True)
return data
def read_data(self, path):
A_path = path + "-A"
B_path = path + "-B"
if self.no_cache:
A_data = indexed_dataset.IndexedDataset(A_path,
fix_lua_indexing=True)
B_data = indexed_dataset.IndexedDataset(B_path,
fix_lua_indexing=True)
else:
A_data = indexed_dataset.IndexedCachedDataset(
A_path, fix_lua_indexing=True)
B_data = indexed_dataset.IndexedCachedDataset(
B_path, fix_lua_indexing=True)
self.a_size = A_data.sizes
self.b_size = B_data.sizes
return (A_data, B_data)
def read_data(self, path):
q_path = path + "-Q"
a_path = path + "-A"
if self.no_cache:
q_data = indexed_dataset.IndexedDataset(q_path,
fix_lua_indexing=True)
a_data = indexed_dataset.IndexedDataset(a_path,
fix_lua_indexing=True)
else:
q_data = indexed_dataset.IndexedCachedDataset(
q_path, fix_lua_indexing=True)
a_data = indexed_dataset.IndexedCachedDataset(
a_path, fix_lua_indexing=True)
self.q_size = q_data.sizes
self.a_size = a_data.sizes
return q_data, a_data