def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
homepage = 'https://raw.githubusercontent.com/google-research-datasets/answer-equivalence-dataset/main/v1/'
archive = {
'train': os.path.join(homepage, 'train.jsonl'),
'ae_dev': os.path.join(homepage, 'ae_dev.jsonl'),
'ae_test': os.path.join(homepage, 'ae_test.jsonl'),
'dev_xlnet': os.path.join(homepage,
'dev_by_system/dev_xlnet.jsonl'),
'dev_luke': os.path.join(homepage, 'dev_by_system/dev_luke.jsonl'),
'dev_bidaf': os.path.join(homepage,
'dev_by_system/dev_bidaf.jsonl')
}
paths = dl_manager.download_and_extract(archive)
return {
split: self._generate_examples(path)
for split, path in paths.items()
}
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
dl_dir = dl_manager.download_and_extract(_LAMBADA_DATASET_URL)
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
gen_kwargs={
'filepath':
os.path.join(dl_dir, 'lambada_development_plain_text.txt')
},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
gen_kwargs={
'filepath': os.path.join(dl_dir, 'lambada_test_plain_text.txt')
},
),
]
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
path = dl_manager.download_and_extract(URL)
subpath = dl_manager.extract(
os.path.join(path, self.builder_config.language + ".tar.gz"))
return [
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
gen_kwargs={"filepath": os.path.join(subpath, "dev")},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
gen_kwargs={"filepath": os.path.join(subpath, "test")},
),
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
gen_kwargs={"filepath": os.path.join(subpath, "train")},
),
]
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
paths = dl_manager.download_and_extract(_URLS)
image_dirs = [
paths['train_images'] / 'train2017',
paths['validation_images'] / 'val2017',
paths['test_images'] / 'test2017',
]
return {
tfds.Split.TRAIN:
self._generate_examples(
image_dirs, paths['train_annotation'] / 'lvis_v1_train.json'),
tfds.Split.VALIDATION:
self._generate_examples(
image_dirs,
paths['validation_annotation'] / 'lvis_v1_val.json'),
tfds.Split.TEST:
self._generate_examples(
image_dirs,
paths['test_annotation'] / 'lvis_v1_image_info_test_dev.json'),
}
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
path = os.path.join(dl_manager.download_and_extract(_DOWNLOAD_URL),
'WikiTableQuestions-master')
examples_sub_dir = os.path.join(path, 'data')
data_splits = {}
for split_name in ['train', 'dev']:
for split_num in range(1, 4):
examples_file = os.path.join(
examples_sub_dir,
f'random-split-{split_num}-{split_name}.tsv')
data_split_key = 'split-{}-{}'.format(split_num, split_name)
data_splits[data_split_key] = self._generate_examples(
examples_path=examples_file, tables_path=path)
data_splits['train'] = self._generate_examples(
examples_path=os.path.join(examples_sub_dir, 'training.tsv'),
tables_path=path)
data_splits['test'] = self._generate_examples(
examples_path=os.path.join(examples_sub_dir,
'pristine-unseen-tables.tsv'),
tables_path=path)
return data_splits
def _split_generators(self, dl_manager: tfds.download.DownloadManager
) -> List[tfds.core.SplitGenerator]:
"""Returns SplitGenerators."""
base_dir = dl_manager.download_and_extract(_DOWNLOAD_URL)
data_dir = os.path.join(base_dir, _DOWNLOAD_ARCHIVE_SUBDIR)
return [
tfds.core.SplitGenerator(
name=tfds.Split.TRAIN,
gen_kwargs={"day_to_paths":
_get_day_to_paths(os.path.join(data_dir, "train"))},
),
tfds.core.SplitGenerator(
name=tfds.Split.VALIDATION,
gen_kwargs={"day_to_paths":
_get_day_to_paths(os.path.join(data_dir, "dev"))},
),
tfds.core.SplitGenerator(
name=tfds.Split.TEST,
gen_kwargs={"day_to_paths":
_get_day_to_paths(os.path.join(data_dir, "test"))},
),
]
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
path = dl_manager.download_and_extract(URL)
return {tfds.Split.TRAIN: self._generate_examples(path)}
def _split_generators(self, dl_manager: tfds.download.DownloadManager):
"""Returns SplitGenerators."""
extracted = dl_manager.download_and_extract(_URLS)
return {k: self._generate_examples(v) for k, v in extracted.items()}