def initialize_plugin(self) -> None:
"""
Code to initialize the plugin
"""
region_name = self.get_config_value("region_name")
aws_access_key_id = self.get_config_value("aws_access_key_id")
aws_secret_access_key = self.get_config_value("aws_secret_access_key")
endpoint_url = self.get_config_value("endpoint_url")
signature_version = self.get_config_value("signature_version")
try:
mirror_base_path = PureS3Path(self.configuration.get("mirror", "directory"))
except (configparser.NoOptionError, configparser.NoSectionError) as e:
logger.error(
"Mirror directory must be set when using s3 as storage backend"
)
raise e
s3_args = {}
if endpoint_url:
s3_args["endpoint_url"] = endpoint_url
if aws_access_key_id:
s3_args["aws_access_key_id"] = aws_access_key_id
if region_name:
s3_args["region_name"] = region_name
if aws_secret_access_key:
s3_args["aws_secret_access_key"] = aws_secret_access_key
if signature_version:
s3_args["config"] = Config(signature_version=signature_version)
resource = boto3.resource("s3", **s3_args)
register_configuration_parameter(mirror_base_path, resource=resource)
def test_boto_methods_with_configuration(s3_mock, reset_configuration_cache):
s3 = boto3.resource('s3')
s3.create_bucket(Bucket='test-bucket')
bucket = S3Path('/test-bucket/')
register_configuration_parameter(bucket, parameters={'ContentType': 'text/html'})
key = bucket.joinpath('bar.html')
key.write_text('hello')
def test_hierarchical_configuration(reset_configuration_cache):
path = S3Path('/foo/')
register_configuration_parameter(path, parameters={'ContentType': 'text/html'})
assert path in _s3_accessor.configuration_map.arguments
assert path not in _s3_accessor.configuration_map.resources
assert _s3_accessor.configuration_map.get_configuration(path) == (
_s3_accessor.configuration_map.default_resource, {'ContentType': 'text/html'})
assert (_s3_accessor.configuration_map.get_configuration(S3Path('/foo/'))
== _s3_accessor.configuration_map.get_configuration(PureS3Path('/foo/')))
def test_open_method_with_custom_endpoint_url():
local_path = PureS3Path('/local/')
register_configuration_parameter(
local_path,
parameters={},
resource=boto3.resource('s3', endpoint_url='http://localhost'))
file_object = S3Path('/local/directory/Test.test').open('br')
if StrictVersion(smart_open.__version__) <= StrictVersion('3.0.0'):
assert file_object._object.meta.client._endpoint.host == 'http://localhost'
else:
assert file_object._client.client._endpoint.host == 'http://localhost'
def s3_init():
minio_bucket_path = PureS3Path('/')
s3 = boto3.resource(
's3',
endpoint_url=os.environ.get("MINIO_URL", 'http://localhost:9000'),
aws_access_key_id=os.environ.get("MINIO_ACCESS_KEY"),
aws_secret_access_key=os.environ.get("MINIO_SECRET_KEY"),
config=Config(signature_version='s3v4'),
region_name='us-east-1')
register_configuration_parameter(minio_bucket_path, resource=s3)
return s3
def s3_mock(reset_configuration_cache: None) -> S3Path:
if os.environ.get("os") != "ubuntu-latest" and os.environ.get("CI"):
pytest.skip("Skip s3 test on non-posix server in github action")
register_configuration_parameter(
PureS3Path("/"),
resource=boto3.resource(
"s3",
aws_access_key_id="minioadmin",
aws_secret_access_key="minioadmin",
endpoint_url="http://localhost:9000",
),
)
new_bucket = S3Path("/test-bucket")
new_bucket.mkdir(exist_ok=True)
yield new_bucket
resource, _ = new_bucket._accessor.configuration_map.get_configuration(new_bucket)
bucket = resource.Bucket(new_bucket.bucket)
for key in bucket.objects.all():
key.delete()
bucket.delete()
def test_configuration_per_bucket(reset_configuration_cache):
local_stack_bucket_path = PureS3Path('/LocalStackBucket/')
minio_bucket_path = PureS3Path('/MinIOBucket/')
default_aws_s3_path = PureS3Path('/')
register_configuration_parameter(default_aws_s3_path,
parameters={'ContentType': 'text/html'})
register_configuration_parameter(local_stack_bucket_path,
parameters={},
resource=boto3.resource(
's3',
endpoint_url='http://localhost:4566'))
register_configuration_parameter(
minio_bucket_path,
parameters={'OutputSerialization': {
'CSV': {}
}},
resource=boto3.resource('s3',
endpoint_url='http://localhost:9000',
aws_access_key_id='minio',
aws_secret_access_key='minio123',
config=Config(signature_version='s3v4'),
region_name='us-east-1'))
assert _s3_accessor.configuration_map.get_configuration(
PureS3Path('/')) == (_s3_accessor.configuration_map.default_resource, {
'ContentType': 'text/html'
})
assert _s3_accessor.configuration_map.get_configuration(
PureS3Path('/some_bucket')) == (
_s3_accessor.configuration_map.default_resource, {
'ContentType': 'text/html'
})
assert _s3_accessor.configuration_map.get_configuration(
PureS3Path('/some_bucket')) == (
_s3_accessor.configuration_map.default_resource, {
'ContentType': 'text/html'
})
resources, arguments = _s3_accessor.configuration_map.get_configuration(
minio_bucket_path)
assert arguments == {'OutputSerialization': {'CSV': {}}}
assert resources.meta.client._endpoint.host == 'http://localhost:9000'
resources, arguments = _s3_accessor.configuration_map.get_configuration(
minio_bucket_path / 'some_key')
assert arguments == {'OutputSerialization': {'CSV': {}}}
assert resources.meta.client._endpoint.host == 'http://localhost:9000'
resources, arguments = _s3_accessor.configuration_map.get_configuration(
local_stack_bucket_path)
assert arguments == {}
assert resources.meta.client._endpoint.host == 'http://localhost:4566'
resources, arguments = _s3_accessor.configuration_map.get_configuration(
local_stack_bucket_path / 'some_key')
assert arguments == {}
assert resources.meta.client._endpoint.host == 'http://localhost:4566'
def test_register_configuration_exceptions(reset_configuration_cache):
with pytest.raises(TypeError):
register_configuration_parameter(
Path('/'), parameters={'ContentType': 'text/html'})
with pytest.raises(TypeError):
register_configuration_parameter(S3Path('/foo/'),
parameters=('ContentType',
'text/html'))
with pytest.raises(ValueError):
register_configuration_parameter(S3Path('/foo/'))
def s3_mock(reset_configuration_cache):
with mock_s3():
register_configuration_parameter(PureS3Path('/'),
resource=boto3.resource('s3'))
yield