def split_exists(split, src, tgt, lang):
filename = os.path.join(
self.args.data, '{}.{}-{}.{}'.format(split, src, tgt, lang))
if self.args.raw_text and IndexedRawTextDataset.exists(filename):
return True
elif not self.args.raw_text and IndexedCachedDataset.exists(
filename):
return True
return False
def indexed_dataset(path):
assert IndexedCachedDataset.exists(
path), f'IndexedCachedDataset.exists({path})'
return IndexedCachedDataset(path, fix_lua_indexing=True)
def indexed_dataset(path, dictionary):
if self.args.raw_text:
return IndexedRawTextDataset(path, dictionary)
elif IndexedCachedDataset.exists(path):
return IndexedDataset(path, fix_lua_indexing=True)
return None