def playwithqueues():
from boto3 import client as botoclient
queue_url = 'https://eu-west-1.queue.amazonaws.com/901610709743/boto_tut_sqs'
sqs_client = botoclient('sqs')
pp = pprint.PrettyPrinter(indent=4)
# Make some message to send to the queue
message_attrs = {
'Title': {
'DataType': 'String',
'StringValue': 'sometitle_tut'
}
}
message_body = ('some_message_body_tut')
# Send the message to the queue
confirmation = write_to_queue(message_attrs,
message_body,
queue_url,
sqs_client,
delay_secs=0)
pp.pprint(confirmation)
# Read the whole queue
pp.pprint(read_queue(queue_url, sqs_client))
def cache_html_templates():
try:
os.mkdir(HTML_TEMPLATE_LOCAL_CACHEDIR, 0o700)
except FileExistsError:
# good.
log.debug('somehow, {} exists already'.format(HTML_TEMPLATE_LOCAL_CACHEDIR))
if os.getenv('HTML_TEMPLATE_DIR', '') == '':
return 'DEFAULT'
bucket = os.getenv('CONFIG_BUCKET')
templatedir = os.getenv('HTML_TEMPLATE_DIR')
if not templatedir[-1] == '/': #we need a trailing slash
templatedir = '{}/'.format(templatedir)
client = botoclient('s3')
try:
result = client.list_objects(Bucket=bucket, Prefix=templatedir, Delimiter='/')
for o in result.get('Contents'):
filename = os.path.basename(o['Key'])
if filename:
log.debug('attempting to save {}'.format(os.path.join(HTML_TEMPLATE_LOCAL_CACHEDIR, filename)))
client.download_file(bucket, o['Key'], os.path.join(HTML_TEMPLATE_LOCAL_CACHEDIR, filename))
return 'CACHED'
except (TypeError, KeyError) as e:
log.error(e)
log.error('Trouble trying to download HTML templates from s3://{}/{}'.format(bucket, templatedir))
return 'ERROR'
def prepare_bucket(name):
"""
Create a writable S3 bucket with a name of `name`. If a writable bucket already exists at that
name, return a Bucket object.
Params:
:name (str) the name of the bucket
Returns:
:str – upon success, returns the bucket name
"""
aws = botoclient('s3')
try:
aws.create_bucket(Bucket=name)
except botocore.exceptions.ClientError as e:
raise e
def get_role_creds(user_id: str = '', in_region: bool = False):
"""
:param user_id: string with URS username
:param in_region: boolean If True a download role that works only in region will be returned
:return: Returns a set of temporary security credentials (consisting of an access key ID, a secret access key, and a security token)
"""
sts = botoclient('sts')
if not user_id:
user_id = 'unauthenticated'
if in_region:
download_role_arn = os.getenv('EGRESS_APP_DOWNLOAD_ROLE_INREGION_ARN')
else:
download_role_arn = os.getenv('EGRESS_APP_DOWNLOAD_ROLE_ARN')
log.debug('assuming role: {}, role session username: {}'.format(
download_role_arn, user_id))
return sts.assume_role(RoleArn=download_role_arn, RoleSessionName=user_id)
def __init__(self, config_file=None, config_name=None): config_file = "./config.json" if config_file is None else config_file config_name = "default" if config_name is None else config_name with open(config_file, "r") as f: config = json.load(f)[config_name] self.service_name = "mturk" self.region_name = "us-east-1" self.aws_access_key_id = config["aws_access_key_id"] self.aws_secret_access_key = config["aws_secret_access_key"] if config["sandbox"]: self.endpoint_url = "https://mturk-requester-sandbox.us-east-1.amazonaws.com" else: self.endpoint_url = "https://mturk-requester.us-east-1.amazonaws.com" self.client = botoclient( service_name = self.service_name, region_name = self.region_name, endpoint_url = self.endpoint_url, aws_access_key_id = self.aws_access_key_id, aws_secret_access_key = self.aws_secret_access_key, )
def is_bucket_writable(name, testkey=TEST_PUT_STUBNAME):
"""
Returns:
tuple: first val is True/False, second val is a dict if success else an error message string
"""
aws = botoclient('s3')
# attempt to put an empty test file
try:
resp = aws.put_object(Bucket=name, Key=testkey)
except botocore.exceptions.ClientError as err:
return (False, str(err))
if resp.get('ResponseMetadata') and resp['ResponseMetadata'].get(
'HTTPStatusCode'):
statuscode = resp['ResponseMetadata']['HTTPStatusCode']
if statuscode == 200:
return (True, resp)
else:
msg = f"HTTP Error; received status code of {statuscode}"
return (False, msg)
else:
return (False, "Error: Did not get HTTPStatusCode")
def __init__(self, region='us-west-2', tag='ecs-services'):
self.region = region
self.tag = tag
self.client = botoclient('ec2', region_name=self.region)
import yaml
from boto3 import resource as botorsc
from boto3 import client as botoclient
with open('../credentials.yml', 'r') as cfile:
cfg = yaml.safe_load(cfile)
aws_region = cfg['aws_region']
aws_key = cfg['aws_key']
aws_access = cfg['aws_access']
departments: list = ['dev', 'test', 'srt']
# Create Client for Session
dynamo_client = botoclient('dynamodb',
aws_access_key_id=aws_key,
aws_secret_access_key=aws_access,
region_name=aws_region)
# Create Amazon resource
dynamo_rsc = botorsc('dynamodb',
aws_access_key_id=aws_key,
aws_secret_access_key=aws_access,
region_name=aws_region)
import logging
import os
import sys
import urllib
from netaddr import IPAddress, IPNetwork
from json import loads
from time import time
from yaml import safe_load
from boto3 import client as botoclient, resource as botoresource, session as botosession, Session as boto_Session
from boto3.resources.base import ServiceResource
from botocore.config import Config as bc_Config
from botocore.exceptions import ClientError
log = logging.getLogger(__name__)
sts = botoclient('sts')
secret_cache = {}
session_cache = {}
region_list_cache = []
s3_resource = None
region = ''
botosess = botosession.Session()
role_creds_cache = {
os.getenv('EGRESS_APP_DOWNLOAD_ROLE_INREGION_ARN'): {},
os.getenv('EGRESS_APP_DOWNLOAD_ROLE_ARN'): {}
}
def get_region():
"""
Will determine and return current AWS region.
:return: string describing AWS region