def split_exists(split, src, tgt, lang): filename = os.path.join( self.args.data, '{}.{}-{}.{}'.format(split, src, tgt, lang)) if self.args.raw_text and IndexedRawTextDataset.exists(filename): return True elif not self.args.raw_text and IndexedCachedDataset.exists( filename): return True return False
def indexed_dataset(path): assert IndexedCachedDataset.exists( path), f'IndexedCachedDataset.exists({path})' return IndexedCachedDataset(path, fix_lua_indexing=True)
def indexed_dataset(path, dictionary): if self.args.raw_text: return IndexedRawTextDataset(path, dictionary) elif IndexedCachedDataset.exists(path): return IndexedDataset(path, fix_lua_indexing=True) return None