def test_connector_args():
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(use_dns_cache=1)
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(keepalive_timeout="1")
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(force_close="1")
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(ssl_context="1")
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# invalid key
connector_args = dict(foo="1")
AioConfig(connector_args)
# test merge
cfg = Config(read_timeout=75)
aio_cfg = AioConfig({'keepalive_timeout': 75})
aio_cfg.merge(cfg)
assert cfg.read_timeout == 75
assert aio_cfg.connector_args['keepalive_timeout'] == 75
def test_merge():
config = AioConfig()
other_config = AioConfig()
new_config = config.merge(other_config)
assert isinstance(new_config, AioConfig)
assert new_config is not config
assert new_config is not other_config
def test_connector_args():
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(use_dns_cache=1)
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(keepalive_timeout="1")
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(force_close="1")
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# wrong type
connector_args = dict(ssl_context="1")
AioConfig(connector_args)
with pytest.raises(ParamValidationError):
# invalid key
connector_args = dict(foo="1")
AioConfig(connector_args)
# test merge
cfg = Config(read_timeout=75)
aio_cfg = AioConfig({'keepalive_timeout': 75})
aio_cfg.merge(cfg)
assert cfg.read_timeout == 75
assert aio_cfg.connector_args['keepalive_timeout'] == 75
def config(region, signature_version):
connect_timeout = read_timout = 5
if _PYCHARM_HOSTED:
connect_timeout = read_timout = 180
return AioConfig(region_name=region, signature_version=signature_version,
read_timeout=read_timout, connect_timeout=connect_timeout)
async def main(loop, args):
session = get_session(loop=loop)
config = AioConfig(max_pool_connections=args.semaphore)
with open(file_name) as fl:
files = [file[:-1] for file in fl.readlines()]
async with session.create_client('s3',
config=config,
aws_access_key_id=args.key_id,
aws_secret_access_key=args.key) as s3_client:
async def get(key):
response = await s3_client.get_object(Bucket=args.bucket,
Key=key)
if response['ResponseMetadata']['HTTPStatusCode'] == 200:
async with response['Body'] as stream:
data = await stream.read()
f = await aio_open(storage_path + '/' + key, mode='w')
await f.write(data.decode('UTF-8'))
await f.flush()
return {key: 'Downloaded'}
else:
return {key: 'Failed'}
return await gather(*[get(file) for file in files])
async def error_coro(queues):
session_1 = aiobotocore.get_session(loop=asyncio.get_running_loop())
session_2 = aiobotocore.get_session(loop=asyncio.get_running_loop())
config = AioConfig(max_pool_connections=AIOBOTO_CONCURRENCY)
async with aiohttp.ClientSession() as download_session, \
session_1.create_client('s3', config=config, **UPLOAD_CREDENTIALS_1) as upload_session_1, \
session_2.create_client('s3', config=config, **UPLOAD_CREDENTIALS_2) as upload_session_2, \
create_engine(**POSTGRESQL_CONNECT_DICT) as database_session:
sessions = dict(
upload=dict(
session=upload_session_1,
host=UPLOAD_CREDENTIALS_1['endpoint_url'],
bucket=BUCKET_NAME_1,
),
upload_2=dict(
session=upload_session_2,
host=UPLOAD_CREDENTIALS_2['endpoint_url'],
bucket=BUCKET_NAME_2,
),
download=download_session,
database=database_session,
)
await image_error_manager(
queues=queues,
sessions=sessions,
)
async def main_coro(queues):
session = aiobotocore.get_session(loop=asyncio.get_running_loop())
config = AioConfig(max_pool_connections=AIOBOTO_CONCURRENCY)
async with aiohttp.ClientSession() as download_session,\
session.create_client('s3', config=config, **UPLOAD_CREDENTIALS_1) as upload_session_1, \
session.create_client('s3', config=config, **UPLOAD_CREDENTIALS_2) as upload_session_2, \
create_engine(**POSTGRESQL_CONNECT_DICT) as database_session:
sessions = dict(
upload=dict(
session=upload_session_1,
host=UPLOAD_CREDENTIALS_1['endpoint_url'],
bucket=BUCKET_NAME_1,
),
upload_2=dict(
session=upload_session_2,
host=UPLOAD_CREDENTIALS_2['endpoint_url'],
bucket=BUCKET_NAME_2,
),
download=download_session,
database=database_session,
)
await publish_msg_to_download_queue(
queues=queues,
sessions=sessions,
page_size=AMOUNT_UPDATE_IMAGES,
)
def test_connector_timeout(loop):
server = AIOServer()
session = AioSession(loop=loop)
config = AioConfig(max_pool_connections=1, connect_timeout=1,
retries={'max_attempts': 0})
s3_client = session.create_client('s3', config=config,
endpoint_url=server.endpoint_url,
aws_secret_access_key='xxx',
aws_access_key_id='xxx')
try:
server.wait_until_up()
@asyncio.coroutine
def get_and_wait():
yield from s3_client.get_object(Bucket='foo', Key='bar')
yield from asyncio.sleep(100)
# this should not raise as we won't have any issues connecting to the
task1 = asyncio.Task(get_and_wait(), loop=loop)
task2 = asyncio.Task(get_and_wait(), loop=loop)
try:
done, pending = yield from asyncio.wait([task1, task2],
timeout=3, loop=loop)
# second request should not timeout just because there isn't a
# connector available
assert len(pending) == 2
finally:
task1.cancel()
task2.cancel()
finally:
s3_client.close()
yield from server.stop()
async def test_connector_timeout(event_loop):
session = AioSession(loop=event_loop)
config = AioConfig(max_pool_connections=1,
connect_timeout=1,
retries={'max_attempts': 0})
async with AIOServer() as server, \
session.create_client('s3', config=config,
endpoint_url=server.endpoint_url,
aws_secret_access_key='xxx',
aws_access_key_id='xxx') as s3_client:
async def get_and_wait():
await s3_client.get_object(Bucket='foo', Key='bar')
await asyncio.sleep(100)
task1 = asyncio.Task(get_and_wait(), loop=event_loop)
task2 = asyncio.Task(get_and_wait(), loop=event_loop)
try:
done, pending = await asyncio.wait([task1, task2],
timeout=3,
loop=event_loop)
# second request should not timeout just because there isn't a
# connector available
assert len(pending) == 2
finally:
task1.cancel()
task2.cancel()
def __init__(self,
nconcurrent=24,
region_name=None,
addressing_style='path'):
from aiobotocore.config import AioConfig
if region_name is None:
region_name = auto_find_region()
s3_cfg = AioConfig(max_pool_connections=nconcurrent,
s3=dict(addressing_style=addressing_style))
self._nconcurrent = nconcurrent
self._async = AsyncThread()
self._s3 = None
self._session = None
self._closed = False
async def setup(s3_cfg):
session = aiobotocore.get_session()
s3 = session.create_client('s3',
region_name=region_name,
config=s3_cfg)
return (session, s3)
session, s3 = self._async.submit(setup, s3_cfg).result()
self._session = session
self._s3 = s3
async def main(args):
session = get_session()
config = AioConfig(max_pool_connections=args.semaphore)
home = path.dirname(path.realpath(__file__))
keys = listdir('./files/')
paths = [home + '/files/' + keys[i] for i in range(0, len(keys))]
async with session.create_client(
's3',
config=config,
aws_access_key_id=args.key_id,
aws_secret_access_key=args.key) as s3_client:
async def put(key, f_path):
f = await aio_open(f_path, mode='r')
fl = await f.read()
await f.close()
response = await s3_client.put_object(Bucket=args.bucket,
Key=key,
Body=fl)
if response['ResponseMetadata']['HTTPStatusCode'] != 200:
return {key: 'Failed'}
else:
return {key: 'Uploaded'}
return await gather(
*[put(keys[i], paths[i]) for i in range(0, len(keys))])
async def test_retry(session: AioSession, caplog):
caplog.set_level(logging.DEBUG)
config = AioConfig(
connect_timeout=1,
read_timeout=1,
# this goes through a slightly different codepath than regular retries
retries={
"mode": "standard",
"total_max_attempts": 3,
},
)
async with session.create_client(
's3',
config=config,
aws_secret_access_key="xxx",
aws_access_key_id="xxx",
endpoint_url='http://localhost:7878') as client:
with pytest.raises(EndpointConnectionError):
await client.get_object(Bucket='foo', Key='bar')
assert 'sleeping for' in caplog.text
async def _get_credentials(self):
"""Get credentials by calling assume role."""
kwargs = self._assume_role_kwargs()
# Assume role with web identity does not require credentials other than
# the token, explicitly configure the client to not sign requests.
config = AioConfig(signature_version=UNSIGNED)
async with self._client_creator('sts', config=config) as client:
return await client.assume_role_with_web_identity(**kwargs)
def config(region, signature_version):
connect_timeout = read_timeout = CONNECT_TIMEOUT
return AioConfig(
region_name=region,
signature_version=signature_version,
read_timeout=read_timeout,
connect_timeout=connect_timeout,
)
async def upload():
async with aioboto3.resource(
's3',
endpoint_url=config.s3.endpoint_url,
aws_access_key_id=config.s3.access_key_id,
aws_secret_access_key=config.s3.secret_access_key.
get_secret_value(),
config=AioConfig(
s3={'addressing_style': config.s3.addressing_style
})) as s3:
bucket = s3.Bucket(config.s3.bucket)
await bucket.upload_file(filename, Path(filename).name)
async def retrieve_with_async(loop, keys, concurrency):
session = aiobotocore.get_session(loop=loop)
config = AioConfig(max_pool_connections=concurrency)
async with session.create_client('s3', config=config) as s3_client:
async def get_and_read(key):
response = await s3_client.get_object(Bucket=BUCKET, Key=key)
async with response['Body'] as stream:
return await stream.read()
return await asyncio.gather(*[get_and_read(key) for key in keys])
def alternative_s3_client(request, session, alternative_region,
signature_version, s3_server,
mocking_test, event_loop):
kw = {}
if mocking_test:
kw = moto_config(s3_server)
config = AioConfig(
region_name=alternative_region, signature_version=signature_version,
read_timeout=5, connect_timeout=5)
client = create_client(
's3', request, event_loop, session, alternative_region, config, **kw)
return client
async def init_client_if_required(self, session=None) -> None:
async with self._init_lock:
if self._client is not None:
return
config = AioConfig(
max_pool_connections=self._max_pool_connections, )
session = session or get_aiosession()
self._client = await self._context_stack.enter_async_context(
session.create_client(self._service_name,
region_name=self._region_name,
config=config))
async def alternative_s3_client(session, alternative_region, signature_version,
s3_server, mocking_test):
kw = moto_config(s3_server) if mocking_test else {}
config = AioConfig(region_name=alternative_region,
signature_version=signature_version,
read_timeout=5,
connect_timeout=5)
async with session.create_client('s3',
region_name=alternative_region,
config=config,
**kw) as client:
yield client
async def test_connector_timeout2(event_loop):
session = AioSession(loop=event_loop)
config = AioConfig(max_pool_connections=1,
connect_timeout=1,
read_timeout=1,
retries={'max_attempts': 0})
async with AIOServer() as server, \
session.create_client('s3', config=config,
endpoint_url=server.endpoint_url,
aws_secret_access_key='xxx',
aws_access_key_id='xxx') as s3_client:
with pytest.raises(ReadTimeoutError):
resp = await s3_client.get_object(Bucket='foo', Key='bar')
await resp["Body"].read()
def __init__(self, cfg: S3Config) -> None:
self.cfg = cfg
self.config = AioConfig(
connector_args={
'keepalive_timeout': cfg.keepalive_timeout,
'use_dns_cache': cfg.use_dns_cache,
'force_close': cfg.force_close,
},
connect_timeout=cfg.connect_timeout,
read_timeout=cfg.read_timeout,
max_pool_connections=cfg.max_pool_connections,
retries={
'max_attempts': cfg.retry_max_attempts,
'mode': cfg.retry_mode,
},
)
self.bucket_name = cfg.bucket_name
self.allowed_types = cfg.allowed_types.split(',')
def __init__(
self,
nconcurrent=24,
region_name=None,
addressing_style="path",
aws_unsigned=None,
):
self._closed = True
if region_name is None:
region_name = auto_find_region()
opts = {}
if aws_unsigned is None:
aws_unsigned = _aws_unsigned_check_env()
if aws_unsigned:
opts["signature_version"] = botocore.UNSIGNED
s3_cfg = AioConfig(
max_pool_connections=nconcurrent,
**opts,
s3=dict(addressing_style=addressing_style),
)
self._nconcurrent = nconcurrent
self._async = AsyncThread()
self._s3 = None
self._s3_ctx = None
self._session = None
async def setup(s3_cfg):
session = get_session()
s3_ctx = session.create_client("s3",
region_name=region_name,
config=s3_cfg)
s3 = await s3_ctx.__aenter__()
return (session, s3, s3_ctx)
session, s3, s3_ctx = self._async.submit(setup, s3_cfg).result()
self._closed = False
self._session = session
self._s3 = s3
self._s3_ctx = s3_ctx
async def run(args):
LOGGER.info('Starting')
# authenticated sessions
wb_session = aiobotocore.AioSession(profile=args.whitebox_profile)
bb_session = aiobotocore.AioSession(profile=args.blackbox_profile)
# unauthenticated session
config = AioConfig(signature_version=UNSIGNED)
session = aiobotocore.get_session()
async with session.create_client('s3',
region_name='us-east-1',
config=config) as anonymous_client:
async with wb_session.create_client(
's3', region_name='us-east-1') as wb_client:
async with bb_session.create_client(
's3', region_name='us-east-1') as bb_client:
with open(
f's3-whitebox_results_{args.whitebox_profile}_{time.strftime("%Y-%m-%d-%H%M%S")}.csv',
'a') as csv_out:
csv_writer = csv.writer(csv_out, delimiter=',')
csv_writer.writerow(
['bucket', 'object', 'content-type', 'url', 'access'])
buckets = await wb_client.list_buckets()
bucket_tasks = []
for bucket in buckets.get('Buckets'):
try:
bucket_location = await wb_client.get_bucket_location(
Bucket=bucket.get('Name'))
bucket_region = bucket_location.get(
'LocationConstraint', 'us-east-1')
if not bucket_region:
bucket_region = 'us-east-1'
except Exception as e:
bucket_region = 'us-east-1'
finally:
bucket_tasks.append(
test_bucket(bucket.get('Name'), bucket_region,
wb_client, bb_client,
anonymous_client, csv_writer))
await asyncio.gather(*bucket_tasks)
LOGGER.info('Done')
def __conn_options(self):
# Memoize the connection options
if self.__memoized_conn_options:
return self.__memoized_conn_options
self.__memoized_conn_options = {
'config': AioConfig(**self.aio_config_params)
}
# This could be blank if the dev wants to use the local auth mechanisms
# for AWS - either environment variables:
# https://boto3.amazonaws.com/v1/documentation/api/latest/
# guide/configuration.html#using-environment-variables
# or a config file at ~/.aws.config:
# https://boto3.amazonaws.com/v1/documentation/api/latest/
# guide/configuration.html#using-a-configuration-file
# Convert these secrets to str to support some secret handlers that
# only provide the values when asked for as strings.
if self.aws_access_key_id:
self.__memoized_conn_options.update({
'aws_access_key_id':
str(self.aws_access_key_id),
'aws_secret_access_key':
str(self.aws_secret_access_key),
})
# Not well hidden, but might as well make it less visible
self.aws_secret_access_key = '(hidden)'
self.aws_access_key_id = '(hidden)'
if self.aws_session_token:
self.__memoized_conn_options['aws_session_token'] = str(
self.aws_session_token)
self.aws_session_token = '(hidden)'
if self.profile_name:
self.__memoized_conn_options['profile_name'] = str(
self.profile_name)
# The endpoint_url isn't part of the configuration.
if self.host_url:
self.__memoized_conn_options['endpoint_url'] = str(self.host_url)
return self.__memoized_conn_options
def __init__(self, app):
if "session" not in app:
loop = app["loop"]
session = get_session(loop=loop)
app["session"] = session
else:
session = app["session"]
self._app = app
if "s3" in app:
if "token_expiration" in app:
# check that our token is not about to expire
expiration = app["token_expiration"]
now = datetime.datetime.now()
delta = expiration - now
if delta.total_seconds() > 10:
self._client = app["s3"]
return
# otherwise, fall through and get a new token
log.info("S3 access token has expired - renewing")
else:
self._client = app["s3"]
return
# first time setup of s3 client or limited time token has expired
aws_region = None
aws_secret_access_key = None
aws_access_key_id = None
aws_iam_role = None
max_pool_connections = 64
aws_session_token = None
try:
aws_iam_role = config.get("aws_iam_role")
except KeyError:
pass
try:
aws_secret_access_key = config.get("aws_secret_access_key")
except KeyError:
pass
try:
aws_access_key_id = config.get("aws_access_key_id")
except KeyError:
pass
try:
aws_region = config.get("aws_region")
except KeyError:
pass
try:
max_pool_connections = config.get('aio_max_pool_connections')
except KeyError:
pass
log.info(f"S3Client init - aws_region {aws_region}")
s3_gateway = config.get('aws_s3_gateway')
if not s3_gateway:
msg = "Invalid aws s3 gateway"
log.error(msg)
raise ValueError(msg)
log.info(f"Using S3Gateway: {s3_gateway}")
use_ssl = False
if s3_gateway.startswith("https"):
use_ssl = True
if not aws_secret_access_key or aws_secret_access_key == 'xxx':
log.info("aws secret access key not set")
aws_secret_access_key = None
if not aws_access_key_id or aws_access_key_id == 'xxx':
log.info("aws access key id not set")
aws_access_key_id = None
if aws_iam_role and not aws_secret_access_key:
log.info(f"using iam role: {aws_iam_role}")
log.info("getting EC2 IAM role credentials")
# Use EC2 IAM role to get credentials
# See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html?icmpid=docs_ec2_console
curl_cmd = [
"curl",
f"http://169.254.169.254/latest/meta-data/iam/security-credentials/{aws_iam_role}"
]
p = subprocess.run(curl_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if p.returncode != 0:
msg = f"Error getting IAM role credentials: {p.stderr}"
log.error(msg)
else:
stdout = p.stdout.decode("utf-8")
try:
cred = json.loads(stdout)
aws_secret_access_key = cred["SecretAccessKey"]
aws_access_key_id = cred["AccessKeyId"]
aws_cred_expiration = cred["Expiration"]
aws_session_token = cred["Token"]
log.info(f"Got Expiration of: {aws_cred_expiration}")
expiration_str = aws_cred_expiration[:-1] + "UTC" # trim off 'Z' and add 'UTC'
# save the expiration
app["token_expiration"] = datetime.datetime.strptime(
expiration_str, "%Y-%m-%dT%H:%M:%S%Z")
except json.JSONDecodeError:
msg = "Unexpected error decoding EC2 meta-data response"
log.error(msg)
except KeyError:
msg = "Missing expected key from EC2 meta-data response"
log.error(msg)
aio_config = AioConfig(max_pool_connections=max_pool_connections)
self._client = session.create_client(
's3',
region_name=aws_region,
aws_secret_access_key=aws_secret_access_key,
aws_access_key_id=aws_access_key_id,
aws_session_token=aws_session_token,
endpoint_url=s3_gateway,
use_ssl=use_ssl,
config=aio_config)
app['s3'] = self._client # save so same client can be returned in subsequent calls
from aiobotocore.config import AioConfig
from os import environ
from contextlib import AsyncExitStack
from starlette.applications import Starlette
from starlette.endpoints import HTTPEndpoint, WebSocketEndpoint
from starlette.responses import JSONResponse
from starlette.routing import Route, WebSocketRoute
LOCALSTACK_HOSTNAME = environ.get("LOCALSTACK_HOSTNAME", "localhost")
LOCALSTACK_ENDPOINT = f"http://{LOCALSTACK_HOSTNAME}:4566"
logger = logging.getLogger()
logger.setLevel(logging.INFO)
config = AioConfig(connect_timeout=10, retries={'max_attempts': 0})
session = aiobotocore.get_session()
_exit_stack = AsyncExitStack()
_db_client = None
async def _create_client(*args, **kwargs):
client = session.create_client(*args, config=config, **kwargs)
client = await _exit_stack.enter_async_context(client)
return client
async def get_aws_db_client():
global _db_client
def config(signature_version):
return AioConfig(signature_version=signature_version,
read_timeout=5,
connect_timeout=5)
},
':route_points': {
'L': extracted
},
}
update_expression = "SET "
if tenant_attr.get('S') is not None:
update_expression += "tenant_id = :tenant_id, "
expression_attribute_values[':tenant_id'] = tenant_attr
update_expression += "route_points = list_append(if_not_exists(route_points, :empty_list), :route_points)"
return update_expression, expression_attribute_values
session = aiobotocore.get_session()
config = AioConfig(max_pool_connections=max_pool_connections)
client = session.create_client('dynamodb',
region_name=boto3.Session().region_name,
config=config)
async def update(dynamo_db_client, update_expression,
expression_attribute_values):
await dynamo_db_client.update_item(
TableName=table_name,
Key={'id': {
'S': get_key()
}},
UpdateExpression=update_expression,
ExpressionAttributeValues=expression_attribute_values,
ReturnValues="NONE")
def resource(self,
service_name,
region_name=None,
api_version=None,
use_ssl=True,
verify=None,
endpoint_url=None,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
config=None):
try:
resource_model = self._loader.load_service_model(
service_name, 'resources-1', api_version)
except UnknownServiceError:
available = self.get_available_resources()
has_low_level_client = (service_name
in self.get_available_services())
raise ResourceNotExistsError(service_name, available,
has_low_level_client)
except DataNotFoundError:
# This is because we've provided an invalid API version.
available_api_versions = self._loader.list_api_versions(
service_name, 'resources-1')
raise UnknownAPIVersionError(service_name, api_version,
', '.join(available_api_versions))
if api_version is None:
# Even though botocore's load_service_model() can handle
# using the latest api_version if not provided, we need
# to track this api_version in boto3 in order to ensure
# we're pairing a resource model with a client model
# of the same API version. It's possible for the latest
# API version of a resource model in boto3 to not be
# the same API version as a service model in botocore.
# So we need to look up the api_version if one is not
# provided to ensure we load the same API version of the
# client.
#
# Note: This is relying on the fact that
# loader.load_service_model(..., api_version=None)
# and loader.determine_latest_version(..., 'resources-1')
# both load the same api version of the file.
api_version = self._loader.determine_latest_version(
service_name, 'resources-1')
# Creating a new resource instance requires the low-level client
# and service model, the resource version and resource JSON data.
# We pass these to the factory and get back a class, which is
# instantiated on top of the low-level client.
if config is not None:
if config.user_agent_extra is None:
config = copy.deepcopy(config)
config.user_agent_extra = 'Resource'
else:
config = AioConfig(user_agent_extra='Resource')
# client = blah part has been moved into a dodgy context class
return ResourceCreaterContext(self, service_name, region_name,
api_version, use_ssl, verify,
endpoint_url, aws_access_key_id,
aws_secret_access_key, aws_session_token,
config, resource_model)
def getS3Client(app):
""" Return s3client handle
"""
if "session" not in app:
# app startup should have set this
raise KeyError("Session not initialized")
session = app["session"]
if "s3" in app:
if "token_expiration" in app:
# check that our token is not about to expire
expiration = app["token_expiration"]
now = datetime.datetime.now()
delta = expiration - now
if delta.total_seconds() > 10:
return app["s3"]
# otherwise, fall through and get a new token
log.info("S3 access token has expired - renewing")
else:
return app["s3"]
# first time setup of s3 client or limited time token has expired
aws_region = config.get("aws_region")
log.info(f"aws_region {aws_region}")
aws_secret_access_key = None
aws_access_key_id = None
aws_session_token = None
aws_iam_role = config.get("aws_iam_role")
aws_secret_access_key = config.get("aws_secret_access_key")
aws_access_key_id = config.get("aws_access_key_id")
if not aws_secret_access_key or aws_secret_access_key == 'xxx':
log.info("aws secret access key not set")
aws_secret_access_key = None
if not aws_access_key_id or aws_access_key_id == 'xxx':
log.info("aws access key id not set")
aws_access_key_id = None
if aws_iam_role and not aws_secret_access_key:
log.info("using iam role: {}".format(aws_iam_role))
log.info("getting EC2 IAM role credentials")
# Use EC2 IAM role to get credentials
# See: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html?icmpid=docs_ec2_console
curl_cmd = ["curl", "http://169.254.169.254/latest/meta-data/iam/security-credentials/{}".format(aws_iam_role)]
p = subprocess.run(curl_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.returncode != 0:
msg = "Error getting IAM role credentials: {}".format(p.stderr)
log.error(msg)
else:
stdout = p.stdout.decode("utf-8")
try:
cred = json.loads(stdout)
aws_secret_access_key = cred["SecretAccessKey"]
aws_access_key_id = cred["AccessKeyId"]
log.info("Got ACCESS_KEY_ID: {} from EC2 metadata".format(aws_access_key_id))
aws_session_token = cred["Token"]
log.info("Got Expiration of: {}".format(cred["Expiration"]))
expiration_str = cred["Expiration"][:-1] + "UTC" # trim off 'Z' and add 'UTC'
# save the expiration
app["token_expiration"] = datetime.datetime.strptime(expiration_str, "%Y-%m-%dT%H:%M:%S%Z")
except json.JSONDecodeError:
msg = "Unexpected error decoding EC2 meta-data response"
log.error(msg)
except KeyError:
msg = "Missing expected key from EC2 meta-data response"
log.error(msg)
s3_gateway = config.get('aws_s3_gateway')
if not s3_gateway:
msg="Invalid aws s3 gateway"
log.error(msg)
raise ValueError(msg)
if s3_gateway[0] == '[' and s3_gateway[-1] == ']':
# convert string to a comma separated list
items = s3_gateway[1:-1].split(',')
s3_gateway = []
for item in items:
s3_gateway.append(item.strip())
if isinstance(s3_gateway, list):
# use the node number to select an item from the list
node_number = 0
if "node_number" in app:
node_number = app["node_number"]
item = s3_gateway[node_number % len(s3_gateway)]
log.debug(f"selecting: {item} from s3_gateway list: {s3_gateway}")
s3_gateway = item
log.info(f"Using S3Gateway: {s3_gateway}")
use_ssl = False
if s3_gateway.startswith("https"):
use_ssl = True
max_pool_connections = config.get('aio_max_pool_connections')
aio_config = AioConfig(max_pool_connections=max_pool_connections)
s3 = session.create_client('s3', region_name=aws_region,
aws_secret_access_key=aws_secret_access_key,
aws_access_key_id=aws_access_key_id,
aws_session_token=aws_session_token,
endpoint_url=s3_gateway,
use_ssl=use_ssl,
config=aio_config)
app['s3'] = s3 # save so same client can be returned in subsequent calls
return s3
def config(region, signature_version):
return AioConfig(region_name=region, signature_version=signature_version,
read_timeout=5, connect_timeout=5)