def mock_as_dataset_base(self, **kwargs):
"""Function which overwrite `builder.as_dataset`."""
# When `USE_FILES` is used, make sure the metadata actually exists.
if tf.io.gfile.exists(self.data_dir):
logging.info('Metadata found for %s at %s', self.name,
self.data_dir)
else:
if policy == MockPolicy.USE_FILES:
raise ValueError(
'TFDS has been mocked with `MockPolicy.USE_FILES`, but metadata '
f'files were not found in {self.data_dir}. '
'You should copy the real metadata files, so that the dataset '
'can be loaded properly, or set the data_dir kwarg of '
'tfds.testing.mock_tfds(data_dir=...).\n')
if policy == MockPolicy.AUTO:
logging.info(
'Metadata NOT found for %s at %s. Will use `MockPolicy.USE_CODE.`',
self.name,
self.data_dir,
)
# Info is already restored at this point, so can mock the file system
# safely in case of `USE_CODE` mode.
# The only gfile access is to check `self.data_dir` existance.
with test_utils.MockFs() as fs:
fs.add_file(os.path.join(self.data_dir, 'tmp.txt'))
return original_as_dataset_fn(self, **kwargs)
def test_mock_fs(self):
fs = test_utils.MockFs()
with fs.mock():
fs.add_file('/path/to/file1', 'Content of file 1')
fs.add_file('/path/file.txt', 'Content of file.txt')
# Test `tf.io.gfile.exists`
self.assertTrue(tf.io.gfile.exists('/path/to/file1'))
self.assertTrue(tf.io.gfile.exists('/path/to/'))
self.assertTrue(tf.io.gfile.exists('/path/to'))
self.assertTrue(tf.io.gfile.exists('/path'))
self.assertFalse(tf.io.gfile.exists('/pat'))
self.assertFalse(tf.io.gfile.exists('/path/to/file1_nonexisting'))
self.assertFalse(tf.io.gfile.exists('/path/to_file1_nonexisting'))
# Test `tf.io.gfile.exists` (relative path)
fs.add_file('relative_path/to/file.txt', 'Content')
self.assertTrue(tf.io.gfile.exists('relative_path/to/file.txt'))
self.assertTrue(tf.io.gfile.exists('relative_path/to/'))
self.assertTrue(tf.io.gfile.exists('relative_path/to'))
self.assertTrue(tf.io.gfile.exists('relative_path'))
self.assertFalse(tf.io.gfile.exists('/relative_path/to'))
# Test `tf.io.gfile.GFile` (write and read mode)
with tf.io.gfile.GFile('/path/to/file2', 'w') as f:
f.write('Content of file 2 (old)')
self.assertEqual(fs.files['/path/to/file2'],
'Content of file 2 (old)')
with tf.io.gfile.GFile('/path/to/file2', 'w') as f:
f.write('Content of file 2 (new)')
self.assertEqual(fs.files['/path/to/file2'],
'Content of file 2 (new)')
with tf.io.gfile.GFile('/path/to/file2', 'r') as f:
self.assertEqual(f.read(), 'Content of file 2 (new)')
# Test `tf.io.gfile.rename`
self.assertEqual(fs.files['/path/to/file1'], 'Content of file 1')
tf.io.gfile.rename('/path/to/file1', '/path/to/file1_moved')
self.assertNotIn('/path/to/file1', fs.files)
self.assertEqual(fs.files['/path/to/file1_moved'],
'Content of file 1')
# Test `tf.io.gfile.listdir`
self.assertCountEqual(tf.io.gfile.listdir('/path/to'),
tf.io.gfile.listdir('/path/to/'))
self.assertCountEqual(tf.io.gfile.listdir('/path/to'),
['file1_moved', 'file2'])
self.assertCountEqual(tf.io.gfile.listdir('/path'),
['file.txt', 'to'])
# Test `MockFs.files`
self.assertEqual(
fs.files, {
'/path/to/file2': 'Content of file 2 (new)',
'/path/to/file1_moved': 'Content of file 1',
'/path/file.txt': 'Content of file.txt',
'relative_path/to/file.txt': 'Content',
})
def test_mock_fs(as_path_fn):
_p = as_path_fn # pylint: disable=invalid-name
fs = test_utils.MockFs()
with fs.mock():
fs.add_file(_p('/path/to/file1'), 'Content of file 1')
fs.add_file(_p('/path/file.txt'), 'Content of file.txt')
# Test `tf.io.gfile.exists`
assert tf.io.gfile.exists(_p('/path/to/file1'))
assert tf.io.gfile.exists(_p('/path/to/'))
assert tf.io.gfile.exists(_p('/path/to'))
assert tf.io.gfile.exists(_p('/path'))
assert not tf.io.gfile.exists(_p('/pat'))
assert not tf.io.gfile.exists(_p('/path/to/file1_nonexisting'))
assert not tf.io.gfile.exists(_p('/path/to_file1_nonexisting'))
# Test `tf.io.gfile.exists` (relative path)
fs.add_file(_p('relative_path/to/file.txt'), 'Content')
assert tf.io.gfile.exists(_p('relative_path/to/file.txt'))
assert tf.io.gfile.exists(_p('relative_path/to/'))
assert tf.io.gfile.exists(_p('relative_path/to'))
assert tf.io.gfile.exists(_p('relative_path'))
assert not tf.io.gfile.exists(_p('/relative_path/to'))
# Test `tf.io.gfile.GFile` (write and read mode)
with tf.io.gfile.GFile(_p('/path/to/file2'), 'w') as f:
f.write('Content of file 2 (old)')
assert fs.files['/path/to/file2'] == 'Content of file 2 (old)'
with tf.io.gfile.GFile(_p('/path/to/file2'), 'w') as f:
f.write('Content of file 2 (new)')
assert fs.files['/path/to/file2'] == 'Content of file 2 (new)'
with tf.io.gfile.GFile(_p('/path/to/file2'), 'r') as f:
assert f.read() == 'Content of file 2 (new)'
# Test `tf.io.gfile.rename`
assert fs.files['/path/to/file1'] == 'Content of file 1'
tf.io.gfile.rename(_p('/path/to/file1'), _p('/path/to/file1_moved'))
assert '/path/to/file1' not in fs.files
assert fs.files['/path/to/file1_moved'] == 'Content of file 1'
# Test `tf.io.gfile.listdir`
assert (
set(tf.io.gfile.listdir(_p('/path/to')))
== set(tf.io.gfile.listdir(_p('/path/to/')))
)
assert set(tf.io.gfile.listdir(_p('/path/to'))) == {'file1_moved', 'file2'}
assert set(tf.io.gfile.listdir(_p('/path'))) == {'file.txt', 'to'}
# Test `MockFs.files`
assert fs.files == {
'/path/to/file2': 'Content of file 2 (new)',
'/path/to/file1_moved': 'Content of file 1',
'/path/file.txt': 'Content of file.txt',
'relative_path/to/file.txt': 'Content',
}
def test_mock_fs(self):
if sys.version_info.major < 3: # Disable test on Python2
return
fs = test_utils.MockFs()
with fs.mock():
fs.add_file('/path/to/file1', 'Content of file 1')
# Test `tf.io.gfile.exists`
self.assertTrue(tf.io.gfile.exists('/path/to/file1'))
self.assertFalse(tf.io.gfile.exists('/path/to/file1_nonexisting'))
# Test `tf.io.gfile.GFile` (write and read mode)
with tf.io.gfile.GFile('/path/to/file2', 'w') as f:
f.write('Content of file 2 (old)')
self.assertEqual(fs.files['/path/to/file2'],
'Content of file 2 (old)')
with tf.io.gfile.GFile('/path/to/file2', 'w') as f:
f.write('Content of file 2 (new)')
self.assertEqual(fs.files['/path/to/file2'],
'Content of file 2 (new)')
with tf.io.gfile.GFile('/path/to/file2', 'r') as f:
self.assertEqual(f.read(), 'Content of file 2 (new)')
# Test `tf.io.gfile.rename`
self.assertEqual(fs.files['/path/to/file1'], 'Content of file 1')
tf.io.gfile.rename('/path/to/file1', '/path/to/file1_moved')
self.assertNotIn('/path/to/file1', fs.files)
self.assertEqual(fs.files['/path/to/file1_moved'],
'Content of file 1')
# Test `tf.io.gfile.listdir`
self.assertEqual(tf.io.gfile.listdir('/path/to'),
tf.io.gfile.listdir('/path/to/'))
self.assertEqual(tf.io.gfile.listdir('/path/to'),
['file2', 'file1_moved'])
# Test `MockFs.files`
self.assertEqual(
fs.files, {
'/path/to/file2': 'Content of file 2 (new)',
'/path/to/file1_moved': 'Content of file 1',
})
