def mock_as_dataset_base(self, **kwargs): """Function which overwrite `builder.as_dataset`.""" # When `USE_FILES` is used, make sure the metadata actually exists. if tf.io.gfile.exists(self.data_dir): logging.info('Metadata found for %s at %s', self.name, self.data_dir) else: if policy == MockPolicy.USE_FILES: raise ValueError( 'TFDS has been mocked with `MockPolicy.USE_FILES`, but metadata ' f'files were not found in {self.data_dir}. ' 'You should copy the real metadata files, so that the dataset ' 'can be loaded properly, or set the data_dir kwarg of ' 'tfds.testing.mock_tfds(data_dir=...).\n') if policy == MockPolicy.AUTO: logging.info( 'Metadata NOT found for %s at %s. Will use `MockPolicy.USE_CODE.`', self.name, self.data_dir, ) # Info is already restored at this point, so can mock the file system # safely in case of `USE_CODE` mode. # The only gfile access is to check `self.data_dir` existance. with test_utils.MockFs() as fs: fs.add_file(os.path.join(self.data_dir, 'tmp.txt')) return original_as_dataset_fn(self, **kwargs)
def test_mock_fs(self): fs = test_utils.MockFs() with fs.mock(): fs.add_file('/path/to/file1', 'Content of file 1') fs.add_file('/path/file.txt', 'Content of file.txt') # Test `tf.io.gfile.exists` self.assertTrue(tf.io.gfile.exists('/path/to/file1')) self.assertTrue(tf.io.gfile.exists('/path/to/')) self.assertTrue(tf.io.gfile.exists('/path/to')) self.assertTrue(tf.io.gfile.exists('/path')) self.assertFalse(tf.io.gfile.exists('/pat')) self.assertFalse(tf.io.gfile.exists('/path/to/file1_nonexisting')) self.assertFalse(tf.io.gfile.exists('/path/to_file1_nonexisting')) # Test `tf.io.gfile.exists` (relative path) fs.add_file('relative_path/to/file.txt', 'Content') self.assertTrue(tf.io.gfile.exists('relative_path/to/file.txt')) self.assertTrue(tf.io.gfile.exists('relative_path/to/')) self.assertTrue(tf.io.gfile.exists('relative_path/to')) self.assertTrue(tf.io.gfile.exists('relative_path')) self.assertFalse(tf.io.gfile.exists('/relative_path/to')) # Test `tf.io.gfile.GFile` (write and read mode) with tf.io.gfile.GFile('/path/to/file2', 'w') as f: f.write('Content of file 2 (old)') self.assertEqual(fs.files['/path/to/file2'], 'Content of file 2 (old)') with tf.io.gfile.GFile('/path/to/file2', 'w') as f: f.write('Content of file 2 (new)') self.assertEqual(fs.files['/path/to/file2'], 'Content of file 2 (new)') with tf.io.gfile.GFile('/path/to/file2', 'r') as f: self.assertEqual(f.read(), 'Content of file 2 (new)') # Test `tf.io.gfile.rename` self.assertEqual(fs.files['/path/to/file1'], 'Content of file 1') tf.io.gfile.rename('/path/to/file1', '/path/to/file1_moved') self.assertNotIn('/path/to/file1', fs.files) self.assertEqual(fs.files['/path/to/file1_moved'], 'Content of file 1') # Test `tf.io.gfile.listdir` self.assertCountEqual(tf.io.gfile.listdir('/path/to'), tf.io.gfile.listdir('/path/to/')) self.assertCountEqual(tf.io.gfile.listdir('/path/to'), ['file1_moved', 'file2']) self.assertCountEqual(tf.io.gfile.listdir('/path'), ['file.txt', 'to']) # Test `MockFs.files` self.assertEqual( fs.files, { '/path/to/file2': 'Content of file 2 (new)', '/path/to/file1_moved': 'Content of file 1', '/path/file.txt': 'Content of file.txt', 'relative_path/to/file.txt': 'Content', })
def test_mock_fs(as_path_fn): _p = as_path_fn # pylint: disable=invalid-name fs = test_utils.MockFs() with fs.mock(): fs.add_file(_p('/path/to/file1'), 'Content of file 1') fs.add_file(_p('/path/file.txt'), 'Content of file.txt') # Test `tf.io.gfile.exists` assert tf.io.gfile.exists(_p('/path/to/file1')) assert tf.io.gfile.exists(_p('/path/to/')) assert tf.io.gfile.exists(_p('/path/to')) assert tf.io.gfile.exists(_p('/path')) assert not tf.io.gfile.exists(_p('/pat')) assert not tf.io.gfile.exists(_p('/path/to/file1_nonexisting')) assert not tf.io.gfile.exists(_p('/path/to_file1_nonexisting')) # Test `tf.io.gfile.exists` (relative path) fs.add_file(_p('relative_path/to/file.txt'), 'Content') assert tf.io.gfile.exists(_p('relative_path/to/file.txt')) assert tf.io.gfile.exists(_p('relative_path/to/')) assert tf.io.gfile.exists(_p('relative_path/to')) assert tf.io.gfile.exists(_p('relative_path')) assert not tf.io.gfile.exists(_p('/relative_path/to')) # Test `tf.io.gfile.GFile` (write and read mode) with tf.io.gfile.GFile(_p('/path/to/file2'), 'w') as f: f.write('Content of file 2 (old)') assert fs.files['/path/to/file2'] == 'Content of file 2 (old)' with tf.io.gfile.GFile(_p('/path/to/file2'), 'w') as f: f.write('Content of file 2 (new)') assert fs.files['/path/to/file2'] == 'Content of file 2 (new)' with tf.io.gfile.GFile(_p('/path/to/file2'), 'r') as f: assert f.read() == 'Content of file 2 (new)' # Test `tf.io.gfile.rename` assert fs.files['/path/to/file1'] == 'Content of file 1' tf.io.gfile.rename(_p('/path/to/file1'), _p('/path/to/file1_moved')) assert '/path/to/file1' not in fs.files assert fs.files['/path/to/file1_moved'] == 'Content of file 1' # Test `tf.io.gfile.listdir` assert ( set(tf.io.gfile.listdir(_p('/path/to'))) == set(tf.io.gfile.listdir(_p('/path/to/'))) ) assert set(tf.io.gfile.listdir(_p('/path/to'))) == {'file1_moved', 'file2'} assert set(tf.io.gfile.listdir(_p('/path'))) == {'file.txt', 'to'} # Test `MockFs.files` assert fs.files == { '/path/to/file2': 'Content of file 2 (new)', '/path/to/file1_moved': 'Content of file 1', '/path/file.txt': 'Content of file.txt', 'relative_path/to/file.txt': 'Content', }
def test_mock_fs(self): if sys.version_info.major < 3: # Disable test on Python2 return fs = test_utils.MockFs() with fs.mock(): fs.add_file('/path/to/file1', 'Content of file 1') # Test `tf.io.gfile.exists` self.assertTrue(tf.io.gfile.exists('/path/to/file1')) self.assertFalse(tf.io.gfile.exists('/path/to/file1_nonexisting')) # Test `tf.io.gfile.GFile` (write and read mode) with tf.io.gfile.GFile('/path/to/file2', 'w') as f: f.write('Content of file 2 (old)') self.assertEqual(fs.files['/path/to/file2'], 'Content of file 2 (old)') with tf.io.gfile.GFile('/path/to/file2', 'w') as f: f.write('Content of file 2 (new)') self.assertEqual(fs.files['/path/to/file2'], 'Content of file 2 (new)') with tf.io.gfile.GFile('/path/to/file2', 'r') as f: self.assertEqual(f.read(), 'Content of file 2 (new)') # Test `tf.io.gfile.rename` self.assertEqual(fs.files['/path/to/file1'], 'Content of file 1') tf.io.gfile.rename('/path/to/file1', '/path/to/file1_moved') self.assertNotIn('/path/to/file1', fs.files) self.assertEqual(fs.files['/path/to/file1_moved'], 'Content of file 1') # Test `tf.io.gfile.listdir` self.assertEqual(tf.io.gfile.listdir('/path/to'), tf.io.gfile.listdir('/path/to/')) self.assertEqual(tf.io.gfile.listdir('/path/to'), ['file2', 'file1_moved']) # Test `MockFs.files` self.assertEqual( fs.files, { '/path/to/file2': 'Content of file 2 (new)', '/path/to/file1_moved': 'Content of file 1', })