Exemple #1
0
 def read_data(self, path):
     if self.no_cache:
         data = indexed_dataset.IndexedDataset(path, fix_lua_indexing=True)
     else:
         data = indexed_dataset.IndexedCachedDataset(path,
                                                     fix_lua_indexing=True)
     return data
Exemple #2
0
 def read_data(self, path):
     A_path = path + "-A"
     B_path = path + "-B"
     if self.no_cache:
         A_data = indexed_dataset.IndexedDataset(A_path,
                                                 fix_lua_indexing=True)
         B_data = indexed_dataset.IndexedDataset(B_path,
                                                 fix_lua_indexing=True)
     else:
         A_data = indexed_dataset.IndexedCachedDataset(
             A_path, fix_lua_indexing=True)
         B_data = indexed_dataset.IndexedCachedDataset(
             B_path, fix_lua_indexing=True)
     self.a_size = A_data.sizes
     self.b_size = B_data.sizes
     return (A_data, B_data)
Exemple #3
0
 def read_data(self, path):
     q_path = path + "-Q"
     a_path = path + "-A"
     if self.no_cache:
         q_data = indexed_dataset.IndexedDataset(q_path,
                                                 fix_lua_indexing=True)
         a_data = indexed_dataset.IndexedDataset(a_path,
                                                 fix_lua_indexing=True)
     else:
         q_data = indexed_dataset.IndexedCachedDataset(
             q_path, fix_lua_indexing=True)
         a_data = indexed_dataset.IndexedCachedDataset(
             a_path, fix_lua_indexing=True)
     self.q_size = q_data.sizes
     self.a_size = a_data.sizes
     return q_data, a_data