def files(self) -> List[Path]:
"""
List the files that compose the dataset.
Returns
-------
List[Path]
List of the paths of all files composing the dataset.
"""
return util.find_files(self.path, FileDataset.is_valid)
def load_json(path: Path, freq: str) -> Iterator[Any]:
for file in find_files(path, FileDataset.is_valid):
for line in open(file):
yield json.loads(line)
def load_json_lines_file(path: Path, freq: str) -> Iterator[Any]:
for file in find_files(path, FileDataset.is_valid):
yield from JsonLinesFile(file)
def baseline(path: Path, freq: str) -> Iterator[Any]:
for file in find_files(path, FileDataset.is_valid):
for line in open(file):
yield line