def test_presigned_invalid_expires(self):
credentials = Credentials(provider=Static())
presign_v4('GET',
'http://localhost:9000/hello',
credentials,
region=None,
headers={},
expires=0)
def parse_minio_credentials(credentials):
if credentials:
access_key = credentials['AccessKeyId']
secret_key = credentials['SecretAccessKey']
session_token = credentials.get('SessionToken', None)
mc_credentials = Credentials(
provider=Static(access_key, secret_key, session_token))
else:
mc_credentials = None
return mc_credentials
def authorize_external_upload(project_id):
if not config.UPLOAD_OBJECT_STORE_ENABLED:
safe_fail_request(
500,
message=
"Retrieving temporary object store credentials feature disabled",
title="Feature Disabled")
headers = request.headers
log, parent_span = bind_log_and_span(project_id)
log.debug("Authorizing external upload")
token = precheck_upload_token(project_id, headers, parent_span)
log.debug(f"Update token is valid")
with db.DBConn() as conn:
dp_id = db.get_dataprovider_id(conn, token)
log = log.bind(dpid=dp_id)
with opentracing.tracer.start_span('assume-role-request',
child_of=parent_span):
client = connect_to_upload_object_store()
client.set_app_info("anonlink", "development version")
bucket_name = config.UPLOAD_OBJECT_STORE_BUCKET
path = object_store_upload_path(project_id, dp_id)
log.info(
f"Retrieving temporary object store credentials for path: '{bucket_name}/{path}'"
)
credentials_provider = AssumeRoleProvider(
client,
Policy=_get_upload_policy(bucket_name, path=path),
DurationSeconds=config.UPLOAD_OBJECT_STORE_STS_DURATION)
credential_values = Credentials(provider=credentials_provider).get()
expiry = credentials_provider._expiry._expiration
log.info("Retrieved temporary credentials")
credentials_json = ObjectStoreCredentials().dump(credential_values)
log.debug("Temp credentials", **credentials_json)
# Convert datetime to ISO 8601 string
credentials_json["Expiration"] = expiry.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
return {
"credentials": credentials_json,
"upload": {
"endpoint": config.UPLOAD_OBJECT_STORE_SERVER,
"secure": config.UPLOAD_OBJECT_STORE_SECURE,
"bucket": bucket_name,
"path": path
}
}, 201
def test_presigned_versioned_id(self):
credentials = Credentials("minio", "minio123")
url = presign_v4(
'GET',
urlsplit(
'http://localhost:9000/bucket-name/objectName?versionId=uuid'),
'us-east-1', credentials, dt, 604800)
eq_(
urlunsplit(url),
'http://localhost:9000/bucket-name/objectName?versionId=uuid&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minio%2F20150620%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20150620T010203Z&X-Amz-Expires=604800&X-Amz-SignedHeaders=host&X-Amz-Signature=3ce13e2ca929fafa20581a05730e4e9435f2a5e20ec7c5a082d175692fb0a663'
)
def test_signv4(self):
# Construct target url.
credentials = Credentials(
provider=Static(
access_key='minio',
secret_key='minio123'
)
)
url = get_target_url('http://localhost:9000', bucket_name='testbucket',
object_name='~testobject', bucket_region='us-east-1', query={'partID': '1', 'uploadID': '~abcd'})
hdrs = sign_v4('PUT', url, 'us-east-1', credentials=credentials, request_datetime=dt)
eq_(hdrs['Authorization'], 'AWS4-HMAC-SHA256 Credential=minio/20150620/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a2f4546f647981732bd90dfa5a7599c44dca92f44bea48ecc7565df06032c25b')
def test_temp_credentials_minio(self):
upload_endpoint = Config.UPLOAD_OBJECT_STORE_SERVER
bucket_name = "uploads"
root_mc_client = connect_to_object_store()
upload_restricted_minio_client = connect_to_upload_object_store()
if not root_mc_client.bucket_exists(bucket_name):
root_mc_client.make_bucket(bucket_name)
with pytest.raises(minio.error.AccessDenied):
upload_restricted_minio_client.list_buckets()
# Should be able to put an object though
upload_restricted_minio_client.put_object(bucket_name,
'testobject',
io.BytesIO(b'data'),
length=4)
credentials_provider = AssumeRoleProvider(
upload_restricted_minio_client, Policy=restricted_upload_policy)
temp_creds = Credentials(provider=credentials_provider)
newly_restricted_mc_client = Minio(upload_endpoint,
credentials=temp_creds,
region='us-east-1',
secure=False)
with pytest.raises(minio.error.AccessDenied):
newly_restricted_mc_client.list_buckets()
# Note this put object worked with the earlier credentials
# But should fail if we have applied the more restrictive policy
with pytest.raises(minio.error.AccessDenied):
newly_restricted_mc_client.put_object(bucket_name,
'testobject2',
io.BytesIO(b'data'),
length=4)
# this path is allowed in the policy however
newly_restricted_mc_client.put_object(bucket_name,
'2020/testobject',
io.BytesIO(b'data'),
length=4)
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A Chain credentials provider, provides a way of chaining multiple providers
# together and will pick the first available using priority order of the
# 'providers' list
from minio import Minio
from minio.credentials import Chain, Credentials, EnvAWS, EnvMinio, IAMProvider
client = Minio('s3.amazonaws.com',
credentials=Credentials(
provider=Chain(
providers=[
IAMProvider(),
EnvAWS(),
EnvMinio()
]
)
))
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# IamEc2MetaData will call AWS metadata service to retrieve credentials
#
# The default AWS metadata service can be found at:
# -> 169.254.169.254/latest/meta-data/iam/security-credentials
#
# If you wish to retrieve credentials from a different place you can provide
# the 'endpoint' paramater to the IamEc2MetaData credentials object
from minio import Minio
from minio.credentials import Credentials, IamEc2MetaData
# Initialize Minio with IamEc2MetaData default credentials object
client = Minio('s3.amazonaws.com',
credentials=Credentials(provider=IamEc2MetaData()))
# Initialize Minio with IamEc2MetaData custom
client = Minio('s3.amazonaws.com',
credentials=Credentials(provider=IamEc2MetaData(
endpoint='custom.endpoint')))
def test_presigned_no_access_key(self):
credentials = Credentials(provider=Static())
presign_v4('GET', 'http://localhost:9000/hello', credentials, None)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# IamEc2MetaData will call AWS metadata service to retrieve credentials
#
# The default AWS metadata service can be found at:
# -> 169.254.169.254/latest/meta-data/iam/security-credentials
#
# If you wish to retrieve credentials from a different place you can provide
# the 'endpoint' paramater to the IamEc2MetaData credentials object
from minio import Minio
from minio.credentials import Credentials, IAMProvider
# Initialize Minio with IamEc2MetaData default credentials object
client = Minio('s3.amazonaws.com',
credentials=Credentials(
provider=IAMProvider()
))
# Initialize Minio with IamEc2MetaData custom
client = Minio('s3.amazonaws.com',
credentials=Credentials(
provider=IAMProvider(endpoint='custom.endpoint')
))
user_client = Minio(
's3.amazonaws.com',
access_key='YOUR-ACCESSKEYID',
secret_key='YOUR-SECRETKEY',
)
_RESTRICTED_UPLOAD_POLICY = """{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:*"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::my-bucket/*"
],
"Sid": "Upload-access-to-specific-bucket-only"
}
]
}
"""
provider = AssumeRoleProvider(
lambda: user_client.get_assume_role_creds(_RESTRICTED_UPLOAD_POLICY), )
client = Minio(
's3.amazonaws.com',
credentials=Credentials(provider),
)
next_retry = retry.increment(
method='GET',
url=url,
error=e
)
if next_retry is None:
global failure
failure = True
raise e
else:
retry = next_retry
logging.warning(f'Retrying download: {retry}')
download_dest.rename(dest)
credentials = Credentials(provider=Static())
client = minio.Minio('s3.amazonaws.com', credentials=credentials)
def get_channels():
return [
(x.object_name, x.last_modified)
for x in client.list_objects_v2('nix-channels')
if re.fullmatch(r'(nixos|nixpkgs)-.+[^/]', x.object_name)
]
def clone_channels():
logging.info(f'- Fetching channels')
channels_to_update = []
working_dir.mkdir(parents=True, exist_ok=True)
secret_key='YOUR-SECRETACCESSKEY')
restricted_upload_policy = """{
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:PutObject"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::uploads/2020/*"
],
"Sid": "Upload-access-to-specific-bucket-only"
}
]
}
"""
credentials_provider = AssumeRoleProvider(client,
Policy=restricted_upload_policy)
temp_creds = Credentials(provider=credentials_provider)
# User can access the credentials for e.g. serialization
print("Retrieved temporary credentials:")
print(temp_creds.get().access_key)
print(temp_creds.get().secret_key)
# Initialize Minio client with the temporary credentials
restricted_client = Minio('localhost:9000', credentials=temp_creds)
# -*- coding: utf-8 -*-
# MinIO Python Library for Amazon S3 Compatible Cloud Storage,
# (C) 2020 MinIO, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A Chain credentials provider, provides a way of chaining multiple providers
# together and will pick the first available using priority order of the
# 'providers' list
from minio import Minio
from minio.credentials import (Chain, Credentials, EnvAWS, EnvMinio,
IamEc2MetaData)
client = Minio('s3.amazonaws.com',
credentials=Credentials(provider=Chain(
providers=[IamEc2MetaData(),
EnvAWS(), EnvMinio()])))
