def setUp(self): super(DatasetBuilderTestCase, self).setUp() self.patchers = [] self.builder = self._make_builder() example_dir = self.DATASET_CLASS.code_path.parent / "dummy_data" if self.EXAMPLE_DIR is not None: self.example_dir = self.EXAMPLE_DIR elif example_dir.exists(): self.example_dir = str(example_dir) else: self.example_dir = os.path.join(test_utils.fake_examples_dir(), self.builder.name) if not tf.io.gfile.exists(self.example_dir): err_msg = ("Dummy data not found in {}." "").format(self.example_dir) if not tf.io.gfile.exists(self.example_dir): # TODO(epot): Better documentation once datasets are migrated to the # folder model. err_msg = "fake_examples dir %s not found." % self.example_dir raise ValueError(err_msg) if self.MOCK_OUT_FORBIDDEN_OS_FUNCTIONS: self._mock_out_forbidden_os_functions() # Track the urls which are downloaded to validate the checksums # The `dl_manager.download` and `dl_manager.download_and_extract` are # patched to record the urls in `_download_urls`. # Calling `dl_manager.download_checksums` stop the url # registration (as checksums are stored remotelly) # `_test_checksums` validates the recorded urls. self._download_urls = set() self._stop_record_download = False
def setUp(self): super(DatasetBuilderTestCase, self).setUp() self.patchers = [] self.builder = self._make_builder() example_dir = self.DATASET_CLASS.code_path.parent / "dummy_data" fake_example_dir = utils.as_path(test_utils.fake_examples_dir()) if self.EXAMPLE_DIR is not None: self.example_dir = utils.as_path(self.EXAMPLE_DIR) example_dir = self.example_dir # Dir to display in the error elif example_dir.exists(): self.example_dir = example_dir else: self.example_dir = fake_example_dir / self.builder.name if not self.example_dir.exists(): err_msg = f"Dummy data not found in: {example_dir}" raise ValueError(err_msg) if self.MOCK_OUT_FORBIDDEN_OS_FUNCTIONS: self._mock_out_forbidden_os_functions() # Track the urls which are downloaded to validate the checksums # The `dl_manager.download` and `dl_manager.download_and_extract` are # patched to record the urls in `_download_urls`. # Calling `dl_manager.download_checksums` stop the url # registration (as checksums are stored remotelly) # `_test_checksums` validates the recorded urls. self._download_urls = set() self._stop_record_download = False
def setUp(self): super(DatasetBuilderTestCase, self).setUp() self.patchers = [] self.builder = self._make_builder() # Determine the fake_examples directory. self.example_dir = os.path.join(test_utils.fake_examples_dir(), self.builder.name) if self.EXAMPLE_DIR is not None: self.example_dir = self.EXAMPLE_DIR if not tf.io.gfile.exists(self.example_dir): err_msg = "fake_examples dir %s not found." % self.example_dir raise ValueError(err_msg) if self.MOCK_OUT_FORBIDDEN_OS_FUNCTIONS: self._mock_out_forbidden_os_functions() # Track the urls which are downloaded to validate the checksums # The `dl_manager.download` and `dl_manager.download_and_extract` are # patched to record the urls in `_download_urls`. # Calling `dl_manager.download_checksums` stop the url # registration (as checksums are stored remotelly) # `_test_checksums` validates the recorded urls. self._download_urls = set() self._stop_record_download = False
def setUp(self): super(DatasetBuilderTestCase, self).setUp() self.patchers = [] self.builder = self._make_builder() # Determine the fake_examples directory. self.example_dir = os.path.join(test_utils.fake_examples_dir(), self.builder.name) if self.EXAMPLE_DIR is not None: self.example_dir = self.EXAMPLE_DIR if not tf.io.gfile.exists(self.example_dir): err_msg = "fake_examples dir %s not found." % self.example_dir raise ValueError(err_msg) if self.MOCK_OUT_FORBIDDEN_OS_FUNCTIONS: self._mock_out_forbidden_os_functions()
def dummy_data(cls) -> epath.Path: # pylint: disable=no-self-argument """Path to the `dummy_data/` directory.""" if cls is DatasetBuilderTestCase: # Required for build_api_docs return None # pytype: disable=bad-return-type dummy_data_expected = cls.DATASET_CLASS.code_path.parent / "dummy_data" fake_example_dir = epath.Path(test_utils.fake_examples_dir()) if cls.EXAMPLE_DIR is not None: dummy_data_found = epath.Path(cls.EXAMPLE_DIR) dummy_data_expected = dummy_data_found # Dir to display in the error elif dummy_data_expected.exists(): dummy_data_found = dummy_data_expected else: dummy_data_found = fake_example_dir / cls.DATASET_CLASS.name if not dummy_data_found.exists(): err_msg = f"Dummy data not found in: {dummy_data_expected}" raise ValueError(err_msg) return dummy_data_found