def test_s3_setup_args():
kwargs = s3_setup_args(S3Config())
assert kwargs == {}
kwargs = s3_setup_args(S3Config(endpoint="http://localhost:30084"))
assert kwargs == {
"client_kwargs": {
"endpoint_url": "http://localhost:30084"
}
}
kwargs = s3_setup_args(S3Config(access_key_id="access"))
assert kwargs == {"key": "access"}
def test_retries(mock_subprocess, mock_check):
mock_subprocess.check_call.side_effect = Exception("test exception (404)")
mock_check.return_value = True
proxy = S3Persistence(data_config=DataConfig(s3=S3Config(backoff=timedelta(
seconds=0))))
assert proxy.exists("s3://test/fdsa/fdsa") is False
assert mock_subprocess.check_call.call_count == 8
def test_get_storage_options():
endpoint = "https://s3.amazonaws.com"
options = get_storage_options(DataConfig(s3=S3Config(endpoint=endpoint)),
"s3://bucket/somewhere")
assert options == {"client_kwargs": {"endpoint_url": endpoint}}
options = get_storage_options(DataConfig(), "/tmp/file")
assert options is None
def test_put_recursive(mock_exec):
proxy = S3Persistence()
proxy.put("/test", "s3://my-bucket/k1", True)
mock_exec.assert_called_with(
cmd=[
"aws", "s3", "cp", "--recursive", "--acl",
"bucket-owner-full-control", "/test", "s3://my-bucket/k1"
],
s3_cfg=S3Config.auto(),
)
def test_get_recursive(mock_exec):
proxy = S3Persistence()
proxy.get("s3://my-bucket/k1", "/test", True)
mock_exec.assert_called_with(
cmd=["aws", "s3", "cp", "--recursive", "s3://my-bucket/k1", "/test"],
s3_cfg=S3Config.auto())
def __init__(self,
default_prefix: Optional[str] = None,
data_config: typing.Optional[DataConfig] = None):
super().__init__(name="awscli-s3", default_prefix=default_prefix)
self.s3_cfg = data_config.s3 if data_config else S3Config.auto()