def test_fs_file_cache_write_binary_data():
with get_temp_dir() as temp_dir:
file_store = FSFileCache(temp_dir)
assert not file_store.has_file_object('foo')
assert file_store.write_binary_data('foo', 'bar'.encode())
file_handle = file_store.get_file_handle('foo')
assert isinstance(file_handle, LocalFileHandle)
assert file_handle.path_desc == os.path.join(temp_dir, 'foo')
def test_fs_file_cache_write_binary_data():
with get_temp_dir() as temp_dir:
file_store = FSFileCache(temp_dir)
assert not file_store.has_file_object("foo")
assert file_store.write_binary_data("foo", b"bar")
file_handle = file_store.get_file_handle("foo")
assert isinstance(file_handle, LocalFileHandle)
assert file_handle.path_desc == os.path.join(temp_dir, "foo")
def test_fs_file_cache_write_data():
bytes_object = io.BytesIO('bar'.encode())
with get_temp_dir() as temp_dir:
file_cache = FSFileCache(temp_dir)
assert not file_cache.has_file_object('foo')
assert file_cache.write_file_object('foo', bytes_object)
file_handle = file_cache.get_file_handle('foo')
assert isinstance(file_handle, LocalFileHandle)
assert file_handle.path_desc == os.path.join(temp_dir, 'foo')
def test_cache_file_from_s3_step_two_skip_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
"boto3.client", new=lambda *_args, **_kwargs: boto_s3
):
execute_solid(
cache_file_from_s3,
ModeDefinition.from_resources({"file_cache": FSFileCache(temp_dir)}),
input_values={"s3_coord": {"bucket": "some-bucket", "key": "some-key"}},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, "some-key"))
def test_cache_file_from_s3_step_two_skip_config():
boto_s3 = mock.MagicMock()
with get_temp_dir() as temp_dir, mock.patch(
'boto3.client', new=lambda *_args, **_kwargs: boto_s3
):
execute_solid(
cache_file_from_s3,
ModeDefinition.from_resources({'file_cache': FSFileCache(temp_dir)}),
input_values={'s3_coord': {'bucket': 'some-bucket', 'key': 'some-key'}},
)
assert boto_s3.download_file.call_count == 1
assert os.path.exists(os.path.join(temp_dir, 'some-key'))
def test_empty_file_cache():
with get_temp_dir() as temp_dir:
file_cache = FSFileCache(temp_dir)
assert not file_cache.has_file_object('kjdfkd')
def unittest_for_local_mode_def(temp_dir, s3_session):
return ModeDefinition.from_resources({'file_cache': FSFileCache(temp_dir), 's3': s3_session})