def test_with_name():
s3_path = PureS3Path('/Downloads/pathlib.tar.gz')
assert s3_path.with_name('setup.py') == PureS3Path('/Downloads/setup.py')
s3_path = PureS3Path('/')
with pytest.raises(ValueError,
message="PureS3Path('/') has an empty name"):
s3_path.with_name('setup.py')
def test_relative_to():
s3_path = PureS3Path('/etc/passwd')
assert s3_path.relative_to('/') == PureS3Path('etc/passwd')
assert s3_path.relative_to('/etc') == PureS3Path('passwd')
with pytest.raises(ValueError,
message="'/etc/passwd' does not start with '/usr'"):
s3_path.relative_to('/usr')
def initialize_plugin(self) -> None:
"""
Code to initialize the plugin
"""
region_name = self.get_config_value("region_name")
aws_access_key_id = self.get_config_value("aws_access_key_id")
aws_secret_access_key = self.get_config_value("aws_secret_access_key")
endpoint_url = self.get_config_value("endpoint_url")
signature_version = self.get_config_value("signature_version")
try:
mirror_base_path = PureS3Path(self.configuration.get("mirror", "directory"))
except (configparser.NoOptionError, configparser.NoSectionError) as e:
logger.error(
"Mirror directory must be set when using s3 as storage backend"
)
raise e
s3_args = {}
if endpoint_url:
s3_args["endpoint_url"] = endpoint_url
if aws_access_key_id:
s3_args["aws_access_key_id"] = aws_access_key_id
if region_name:
s3_args["region_name"] = region_name
if aws_secret_access_key:
s3_args["aws_secret_access_key"] = aws_secret_access_key
if signature_version:
s3_args["config"] = Config(signature_version=signature_version)
resource = boto3.resource("s3", **s3_args)
register_configuration_parameter(mirror_base_path, resource=resource)
def test_hierarchical_configuration(reset_configuration_cache):
path = S3Path('/foo/')
register_configuration_parameter(path, parameters={'ContentType': 'text/html'})
assert path in _s3_accessor.configuration_map.arguments
assert path not in _s3_accessor.configuration_map.resources
assert _s3_accessor.configuration_map.get_configuration(path) == (
_s3_accessor.configuration_map.default_resource, {'ContentType': 'text/html'})
assert (_s3_accessor.configuration_map.get_configuration(S3Path('/foo/'))
== _s3_accessor.configuration_map.get_configuration(PureS3Path('/foo/')))
def preview(self):
print("hi?", request.path)
path = ""
response = Response()
if "/s3buckets/" not in request.path:
path = request.path
else:
s3_client = boto3.client('s3')
p = PureS3Path(request.path.replace("/s3buckets", ""))
# print (p)
# print ("lol", p.bucket, p.key)
bucketname = str(p.bucket).replace("/", '')
itemkey = str(p.key)
path = s3_client.generate_presigned_url('get_object',
Params={
'Bucket': bucketname,
'Key': itemkey
},
ExpiresIn=100)
# print (path)
response.data = """
<html>
<head>
<style>
* {
border: 0px;
margin: 0px;
padding: 0px;
}
body {
background: black;
}
img {
max-width: 100%%;
max-height: 100%%;
margin: auto;
display: block;
position: absolute;
top: 50%%;
bottom: 50%%;
}
</style>
</head>
<body>
<img src="%s?dl">
</body>
</html>
""" % request.path
return response
def test_open_method_with_custom_endpoint_url():
local_path = PureS3Path('/local/')
register_configuration_parameter(
local_path,
parameters={},
resource=boto3.resource('s3', endpoint_url='http://localhost'))
file_object = S3Path('/local/directory/Test.test').open('br')
if StrictVersion(smart_open.__version__) <= StrictVersion('3.0.0'):
assert file_object._object.meta.client._endpoint.host == 'http://localhost'
else:
assert file_object._client.client._endpoint.host == 'http://localhost'
def test_with_suffix():
s3_path = PureS3Path('/Downloads/pathlib.tar.gz')
assert s3_path.with_suffix('.bz2') == PureS3Path('/Downloads/pathlib.tar.bz2')
s3_path = PureS3Path('README')
assert s3_path.with_suffix('.txt') == PureS3Path('README.txt')
s3_path = PureS3Path('README.txt')
assert s3_path.with_suffix('') == PureS3Path('README')
def test_drive():
assert PureS3Path('foo//bar').drive == ''
assert PureS3Path('foo/./bar').drive == ''
assert PureS3Path('foo/../bar').drive == ''
assert PureS3Path('../bar').drive == ''
assert PureS3Path('foo', '../bar').drive == ''
assert PureS3Path('/foo/bar').drive == ''
def test_match():
assert PureS3Path('a/b.py').match('*.py')
assert PureS3Path('/a/b/c.py').match('b/*.py')
assert not PureS3Path('/a/b/c.py').match('a/*.py')
assert PureS3Path('/a.py').match('/*.py')
assert not PureS3Path('a/b.py').match('/*.py')
assert not PureS3Path('a/b.py').match('*.Py')
def test_root():
assert PureS3Path('foo//bar').root == ''
assert PureS3Path('foo/./bar').root == ''
assert PureS3Path('foo/../bar').root == ''
assert PureS3Path('../bar').root == ''
assert PureS3Path('foo', '../bar').root == ''
assert PureS3Path('/foo/bar').root == '/'
def test_anchor():
assert PureS3Path('foo//bar').anchor == ''
assert PureS3Path('foo/./bar').anchor == ''
assert PureS3Path('foo/../bar').anchor == ''
assert PureS3Path('../bar').anchor == ''
assert PureS3Path('foo', '../bar').anchor == ''
assert PureS3Path('/foo/bar').anchor == '/'
def test_parts():
assert PureS3Path('foo//bar').parts == ('foo', 'bar')
assert PureS3Path('foo/./bar').parts == ('foo', 'bar')
assert PureS3Path('foo/../bar').parts == ('bar',)
assert PureS3Path('../bar').parts == ('..', 'bar')
assert PureS3Path('foo', '../bar').parts == ('bar',)
assert PureS3Path('/foo/bar').parts == ('/', 'foo', 'bar')
def test_basic_configuration(reset_configuration_cache):
path = S3Path('/foo/')
_s3_accessor.configuration_map.arguments = _s3_accessor.configuration_map.resources = None
assert path not in (_s3_accessor.configuration_map.arguments or ())
assert path not in (_s3_accessor.configuration_map.resources or ())
assert _s3_accessor.configuration_map.get_configuration(path) == (
_s3_accessor.configuration_map.default_resource, {})
assert (_s3_accessor.configuration_map.get_configuration(
S3Path('/foo/')) == _s3_accessor.configuration_map.get_configuration(
PureS3Path('/foo/')))
def s3_init():
minio_bucket_path = PureS3Path('/')
s3 = boto3.resource(
's3',
endpoint_url=os.environ.get("MINIO_URL", 'http://localhost:9000'),
aws_access_key_id=os.environ.get("MINIO_ACCESS_KEY"),
aws_secret_access_key=os.environ.get("MINIO_SECRET_KEY"),
config=Config(signature_version='s3v4'),
region_name='us-east-1')
register_configuration_parameter(minio_bucket_path, resource=s3)
return s3
def test_configuration_per_bucket(reset_configuration_cache):
local_stack_bucket_path = PureS3Path('/LocalStackBucket/')
minio_bucket_path = PureS3Path('/MinIOBucket/')
default_aws_s3_path = PureS3Path('/')
register_configuration_parameter(default_aws_s3_path,
parameters={'ContentType': 'text/html'})
register_configuration_parameter(local_stack_bucket_path,
parameters={},
resource=boto3.resource(
's3',
endpoint_url='http://localhost:4566'))
register_configuration_parameter(
minio_bucket_path,
parameters={'OutputSerialization': {
'CSV': {}
}},
resource=boto3.resource('s3',
endpoint_url='http://localhost:9000',
aws_access_key_id='minio',
aws_secret_access_key='minio123',
config=Config(signature_version='s3v4'),
region_name='us-east-1'))
assert _s3_accessor.configuration_map.get_configuration(
PureS3Path('/')) == (_s3_accessor.configuration_map.default_resource, {
'ContentType': 'text/html'
})
assert _s3_accessor.configuration_map.get_configuration(
PureS3Path('/some_bucket')) == (
_s3_accessor.configuration_map.default_resource, {
'ContentType': 'text/html'
})
assert _s3_accessor.configuration_map.get_configuration(
PureS3Path('/some_bucket')) == (
_s3_accessor.configuration_map.default_resource, {
'ContentType': 'text/html'
})
resources, arguments = _s3_accessor.configuration_map.get_configuration(
minio_bucket_path)
assert arguments == {'OutputSerialization': {'CSV': {}}}
assert resources.meta.client._endpoint.host == 'http://localhost:9000'
resources, arguments = _s3_accessor.configuration_map.get_configuration(
minio_bucket_path / 'some_key')
assert arguments == {'OutputSerialization': {'CSV': {}}}
assert resources.meta.client._endpoint.host == 'http://localhost:9000'
resources, arguments = _s3_accessor.configuration_map.get_configuration(
local_stack_bucket_path)
assert arguments == {}
assert resources.meta.client._endpoint.host == 'http://localhost:4566'
resources, arguments = _s3_accessor.configuration_map.get_configuration(
local_stack_bucket_path / 'some_key')
assert arguments == {}
assert resources.meta.client._endpoint.host == 'http://localhost:4566'
def s3_mock(reset_configuration_cache: None) -> S3Path:
if os.environ.get("os") != "ubuntu-latest" and os.environ.get("CI"):
pytest.skip("Skip s3 test on non-posix server in github action")
register_configuration_parameter(
PureS3Path("/"),
resource=boto3.resource(
"s3",
aws_access_key_id="minioadmin",
aws_secret_access_key="minioadmin",
endpoint_url="http://localhost:9000",
),
)
new_bucket = S3Path("/test-bucket")
new_bucket.mkdir(exist_ok=True)
yield new_bucket
resource, _ = new_bucket._accessor.configuration_map.get_configuration(new_bucket)
bucket = resource.Bucket(new_bucket.bucket)
for key in bucket.objects.all():
key.delete()
bucket.delete()
def test_join_paths():
assert PureS3Path(Path('foo'), Path('bar')) == PureS3Path('foo/bar')
def test_paths_of_a_different_flavour():
with pytest.raises(TypeError):
PureS3Path('/bucket/key') < PurePosixPath('/bucket/key')
with pytest.raises(TypeError):
PureWindowsPath('/bucket/key')> PureS3Path('/bucket/key')
def test_paths_of_a_different_flavour():
with pytest.raises(TypeError, message="'<' not supported between instances of 'PureS3Path' and 'PurePosixPath'"):
PureS3Path('/bucket/key') < PurePosixPath('/bucket/key')
with pytest.raises(TypeError, message="'>' not supported between instances of 'PureWindowsPath' and 'PureS3Path'"):
PureWindowsPath('/bucket/key')> PureS3Path('/bucket/key')
"""
A lot of what you see here should be passed to the notebook at runtime.
"""
from pathlib import Path
import os
from s3path import PureS3Path
import boto3
from botocore.exceptions import ClientError
import sagemaker
# Main names and paths
model_name = "aamod20200130"
data_dir = Path("./data/outputs/")
s3_root = PureS3Path('/robtests/sagemaker/aa/')
# train/test/validate dirs in S3
s3_train_loc = s3_root / "train"
s3_test_loc = s3_root / "test"
s3_validation_loc = s3_root / "validation"
# Predict in/out in s3
s3_predict_in = s3_root / 'predictions_in'
s3_predict_out = s3_root / 'predictions_out'
# Boto3 connection (this handles talking to AWS)
os.environ['AWS_DEFAULT_REGION'] = "us-east-1"
boto_session = boto3.session.Session(profile_name='saml')
# sagemaker specific AWS interfaces
from pathlib import Path
import os
from s3path import PureS3Path
import boto3
from botocore.exceptions import ClientError
import sagemaker
# Main names and paths
model_name = "miguel-learning-xgboost-4"
data_dir = Path("YOUR_LOCAL_DATA_DIR")
s3_root = PureS3Path('/miguel-sagemaker/')
# data dirs in S3
s3_data_loc = s3_root / "data"
s3_model_loc = s3_root / "model"
# batch output
s3_predict_out = s3_data_loc / 'predictions'
# Boto3 connection (this handles talking to AWS)
os.environ['AWS_DEFAULT_REGION'] = "us-east-1"
boto_session = boto3.session.Session(profile_name='saml')
# sagemaker specific AWS interfaces
sm_sess = sagemaker.Session(boto_session=boto_session)
sm_client = boto_session.client('sagemaker')
# role to define my permissions on AWS, autopopulates if you are on sagemaker.
try:
role = sagemaker.get_execution_role()
def test_parents():
assert tuple(PureS3Path('foo//bar').parents) == (PureS3Path('foo'), PureS3Path('.'))
assert tuple(PureS3Path('foo/./bar').parents) == (PureS3Path('foo'), PureS3Path('.'))
assert tuple(PureS3Path('foo/../bar').parents) == (PureS3Path('.'),)
assert tuple(PureS3Path('../bar').parents) == (PureS3Path('..'), PureS3Path('.'))
assert tuple(PureS3Path('foo', '../bar').parents) == (PureS3Path('.'),)
assert tuple(PureS3Path('/foo/bar').parents) == (PureS3Path('/foo'), PureS3Path('/'))
def test_joinpath():
assert PureS3Path('/etc').joinpath('passwd') == PureS3Path('/etc/passwd')
assert PureS3Path('/etc').joinpath(PureS3Path('passwd')) == PureS3Path('/etc/passwd')
assert PureS3Path('/etc').joinpath('init.d', 'apache2') == PureS3Path('/etc/init.d/apache2')
def test_relative_to():
s3_path = PureS3Path('/etc/passwd')
assert s3_path.relative_to('/') == PureS3Path('etc/passwd')
assert s3_path.relative_to('/etc') == PureS3Path('passwd')
with pytest.raises(ValueError):
s3_path.relative_to('/usr')
def test_fspath():
assert os.fspath(PureS3Path('/usr/bin')) == '/usr/bin'
def test_operators():
assert PureS3Path('/etc') / 'init.d' / 'apache2' == PureS3Path('/etc/init.d/apache2')
assert '/usr' / PureS3Path('bin') == PureS3Path('/usr/bin')
def test_slashes_single_double_dots():
assert PureS3Path('foo//bar') == PureS3Path('foo/bar')
assert PureS3Path('foo/./bar') == PureS3Path('foo/bar')
assert PureS3Path('foo/../bar') == PureS3Path('bar')
assert PureS3Path('../bar') == PureS3Path('../bar')
assert PureS3Path('foo', '../bar') == PureS3Path('bar')
def test_absolute_paths():
assert PureS3Path('/etc', '/usr', 'lib64') == PureS3Path('/usr/lib64')
def test_empty():
assert PureS3Path() == PureS3Path('.')
def test_join_strs():
assert PureS3Path('foo', 'some/path', 'bar') == PureS3Path('foo/some/path/bar')