import os from argparse import ArgumentParser from pathlib import Path from shutil import copyfile, rmtree from typing import Generator from datasets.builder import FORCE_REDOWNLOAD, REUSE_CACHE_IF_EXISTS, DatasetBuilder from datasets.commands import BaseTransformersCLICommand from datasets.info import DATASET_INFOS_DICT_FILE_NAME from datasets.load import import_main_class, prepare_module from datasets.utils.filelock import logger as fl_logger from datasets.utils.logging import ERROR, get_logger logger = get_logger(__name__) def test_command_factory(args): return TestCommand( args.dataset, args.name, args.cache_dir, args.data_dir, args.all_configs, args.save_infos, args.ignore_verifications, args.force_redownload, args.clear_cache, args.proc_rank, args.num_proc, )
def __init__(self, tfds_path: str, datasets_directory: str, *args): self._logger = get_logger("datasets-cli/converting") self._tfds_path = tfds_path self._datasets_directory = datasets_directory