def deliver_sns_message(self, topic, subject, rendered_jinja_body, sqs_message):
# Max length of subject in sns is 100 chars
if len(subject) > 100:
subject = subject[:97] + '..'
try:
account = topic.split(':')[4]
if account in self.sns_cache:
sns = self.sns_cache[account]
else:
# if cross_accounts isn't set, we'll try using the current credentials
if account not in self.config.get('cross_accounts', []):
session = Session()
else:
creds = self.aws_sts.assume_role(
RoleArn=self.config['cross_accounts'][account],
RoleSessionName="CustodianNotification")['Credentials']
session = Session(
aws_access_key_id=creds['AccessKeyId'],
aws_secret_access_key=creds['SecretAccessKey'],
aws_session_token=creds['SessionToken'])
self.sns_cache[account] = sns = session.client('sns')
self.logger.info("Sending account:%s policy:%s sns:%s to %s" % (
sqs_message.get('account', ''),
sqs_message['policy']['name'],
sqs_message['action'].get('template', 'default'),
topic))
sns.publish(TopicArn=topic, Subject=subject, Message=rendered_jinja_body)
except Exception as e:
self.logger.warning(
"Error policy:%s account:%s sending sns to %s \n %s" % (
sqs_message['policy'], sqs_message.get('account', 'na'), topic, e))
def configure_sqs_client(graph):
endpoint_url = graph.config.sqs_consumer.endpoint_url
profile_name = graph.config.sqs_consumer.profile_name
region_name = graph.config.sqs_consumer.region_name
session = Session(profile_name=profile_name)
return session.client(
"sqs",
endpoint_url=endpoint_url,
region_name=region_name,
)
class AWSClient(object):
"""Manages automatically creating and destroying clients to AWS services."""
def __init__(self, resource, config, credentials=None, region_name=None):
"""Constructor
:param resource: AWS specific token for resource type. e.g., 's3', 'sqs', etc.
:type resource: string
:param config: Resource specific configuration
:type config: :class:`botocore.client.Config`
:param credentials: Authentication values needed to access AWS. If no credentials are passed, then IAM
role-based access is assumed.
:type credentials: :class:`util.aws.AWSCredentials`
:param region_name: The AWS region the resource resides in.
:type region_name: string
"""
self.credentials = credentials
self.region_name = region_name
self._client = None
self._resource_name = resource
self._config = config
def __enter__(self):
"""Callback handles creating a new client for AWS access."""
logger.debug('Setting up AWS client...')
session_args = {}
if self.credentials:
session_args['aws_access_key_id'] = self.credentials.access_key_id
session_args['aws_secret_access_key'] = self.credentials.secret_access_key
if self.region_name:
session_args['region_name'] = self.region_name
self._session = Session(**session_args)
self._client = self._session.client(self._resource_name, config=self._config)
self._resource = self._session.resource(self._resource_name, config=self._config)
return self
def __exit__(self, type, value, traceback):
"""Callback handles destroying an existing client."""
pass
@staticmethod
def instantiate_credentials_from_config(config):
if 'credentials' in config and config['credentials']:
credentials_dict = config['credentials']
if 'access_key_id' not in credentials_dict or not credentials_dict['access_key_id']:
raise InvalidAWSCredentials('"credentials" requires "access_key_id" to be populated')
if 'secret_access_key' not in credentials_dict or not credentials_dict['secret_access_key']:
raise InvalidAWSCredentials('"credentials" requires "secret_access_key" to be populated')
return AWSCredentials(credentials_dict['access_key_id'], credentials_dict['secret_access_key'])
def decrypt(self, value, context=None):
if not context:
context = {}
session = Session(profile_name=self.profile_name)
kms = session.client('kms', region_name=self.region)
key_service = KeyService(kms, self.kms_key, context)
return open_aes_ctr_legacy(
key_service,
dict(
key=value.key,
contents=value.contents,
hmac=value.hmac,
)
)
def configure_sns_producer(graph):
"""
Configure an SNS producer.
The SNS Producer requires the following collaborators:
- Opaque from microcosm.opaque for capturing context information
- an aws sns client, i.e. from boto.
- pubsub message codecs: see tests for examples.
- sns topic arns: see tests for examples.
"""
if graph.metadata.testing:
from unittest.mock import MagicMock
if not graph.config.sns_producer.mock_sns:
return MagicMock()
sns_client = MagicMock()
else:
endpoint_url = graph.config.sns_producer.endpoint_url
profile_name = graph.config.sns_producer.profile_name
region_name = graph.config.sns_producer.region_name
session = Session(profile_name=profile_name)
sns_client = session.client(
"sns",
endpoint_url=endpoint_url,
region_name=region_name,
)
try:
opaque = graph.opaque
except NotBoundError:
opaque = None
if graph.config.sns_producer.skip is None:
# In development mode, default to not publishing because there's typically
# not anywhere to publish to (e.g. no SNS topic)
skip = graph.metadata.debug
else:
# If configured explicitly, respect the flag
skip = strtobool(graph.config.sns_producer.skip)
return SNSProducer(
opaque=opaque,
pubsub_message_schema_registry=graph.pubsub_message_schema_registry,
sns_client=sns_client,
sns_topic_arns=graph.sns_topic_arns,
skip=skip,
deferred_batch_size=graph.config.sns_producer.deferred_batch_size,
)
def encrypt(self, plaintext, context=None):
if not context:
context = {}
session = Session(profile_name=self.profile_name)
kms = session.client('kms', region_name=self.region)
key_service = KeyService(kms, self.kms_key, context)
sealed = seal_aes_ctr_legacy(
key_service,
plaintext,
)
return EncryptedValue(
sealed["key"],
sealed["contents"],
sealed["hmac"],
)
class DatabaseResource(object):
def __init__(self):
self.session = Session(profile_name=profile_name, region_name="us-east-1")
self.content_type = "application/json"
def on_get(self, request, response):
rds = self.session.client('rds')
databases = rds.describe_db_instances()
for instance in databases['DBInstances']:
instance['InstanceCreateTime'] = str(instance["InstanceCreateTime"])
instance['LatestRestorableTime'] = str(instance["LatestRestorableTime"])
response.status = falcon.HTTP_200
response.content_type = self.content_type
response.body = json.dumps(databases)
class CacheResource(object):
def __init__(self):
self.session = Session(profile_name=profile_name, region_name="us-east-1")
self.content_type = "application/json"
def on_get(self, request, response):
redis = self.session.client('elasticache')
clusters = redis.describe_cache_clusters(ShowCacheNodeInfo=True)
for cluster in clusters['CacheClusters']:
cluster['CacheClusterCreateTime'] = str(cluster['CacheClusterCreateTime'])
for node in cluster["CacheNodes"]:
node['CacheNodeCreateTime'] = str(node['CacheNodeCreateTime'])
response.status = falcon.HTTP_200
response.content_type = self.content_type
response.body = json.dumps(clusters)
def run():
cfg = Config(region_name="us-east-1",
parameter_validation=False,
max_pool_connections=10)
session = Session()
s3 = session.client('s3', config=cfg)
response = s3.select_object_content(
Bucket='s3filter',
Key='parquet/supplier.large.csv',
Expression='select sum(cast(s_acctbal as float)) from s3Object',
ExpressionType='SQL',
InputSerialization={
'CompressionType': 'NONE',
'CSV': {
'FileHeaderInfo': 'Use',
'RecordDelimiter': '\n',
'FieldDelimiter': '|'
}
},
OutputSerialization={'CSV': {}})
df = None
cursor = PandasCursor(None)
cursor.event_stream = response['Payload']
dfs = cursor.parse_event_stream()
for partial_df in dfs:
if df is None:
df = partial_df
else:
df = pd.concat(df, partial_df)
assert len(df) == 1
assert pd.to_numeric(
df.iloc[0]['_0']) == pytest.approx(22551774325.00404)
print("{} | {}".format(test_s3_select_from_large_csv.__name__,
cursor.bytes_scanned))
print("{} | {}".format(test_s3_select_from_large_csv.__name__,
cursor.bytes_processed))
print("{} | {}".format(test_s3_select_from_large_csv.__name__,
cursor.bytes_returned))
def get_instance_address(
session: boto3.Session,
instance_id: str,
) -> Tuple[str, str, str]:
"""
Create a temporary publicly accessible IP for the given instance.
Return the IP address, the allocation id, and the association id.
"""
client = session.client("ec2")
allocation_response = client.allocate_address(
Domain="vpc",
TagSpecifications=[{
"ResourceType":
"elastic-ip",
"Tags": [
{
"Key": "Name",
"Value": f"{instance_id}-ip-address",
},
get_owner_tag(),
],
}],
)
ip_address = allocation_response["PublicIp"]
allocation_id = allocation_response["AllocationId"]
associate_response = client.associate_address(
AllocationId=allocation_id,
InstanceId=instance_id,
AllowReassociation=False,
)
association_id = associate_response["AssociationId"]
# Remove this IP from known hosts in case it's there,
# as it's definitely not the old host anymore
subprocess.check_call([
"ssh-keygen",
"-f",
f"{KNOWN_HOST_PATH}",
"-R",
f'"{ip_address}"',
])
return ip_address, allocation_id, association_id
def configure_base_balancer(
session: boto3.Session,
balancer_arn: str,
certificate_arn: str,
target_group_arn: str,
) -> str:
"""
Configure the default rules for this load balancer. Return the id
of the listener to add rules to for redirecting to specified target groups
"""
client = session.client("elbv2")
_redirect_response = client.create_listener(
LoadBalancerArn=balancer_arn,
Protocol="HTTP",
Port=80,
DefaultActions=[{
"Type": "redirect",
"RedirectConfig": {
"Protocol": "HTTPS",
"Port": "443",
"Host": "#{host}",
"Path": "/#{path}",
"Query": "#{query}",
"StatusCode": "HTTP_301",
},
}],
)
forward_response = client.create_listener(
LoadBalancerArn=balancer_arn,
Protocol="HTTPS",
Port=443,
SslPolicy="ELBSecurityPolicy-2016-08",
Certificates=[{
"CertificateArn": certificate_arn,
}],
DefaultActions=[{
"Type": "forward",
"TargetGroupArn": target_group_arn,
}],
)
listener_arn = forward_response["Listeners"][0]["ListenerArn"]
return listener_arn
def speak(self, text):
session = Session(profile_name="default")
polly = session.client("polly")
try:
# Request speech synthesis
response = polly.synthesize_speech(Text=text,
OutputFormat="mp3",
VoiceId="Joanna")
except (BotoCoreError, ClientError) as error:
# The service returned an error, exit gracefully
print(error)
sys.exit(-1)
# Access the audio stream from the response
if "AudioStream" in response:
# Note: Closing the stream is important as the service throttles on the
# number of parallel connections. Here we are using contextlib.closing to
# ensure the close method of the stream object will be called automatically
# at the end of the with statement's scope.
with closing(response["AudioStream"]) as stream:
output = os.path.join(gettempdir(), "speech.mp3")
try:
# Open a file for writing the output as a binary stream
with open(output, "wb") as file:
file.write(stream.read())
except IOError as error:
# Could not write to file, exit gracefully
print(error)
sys.exit(-1)
else:
# The response didn't contain audio data, exit gracefully
print("Could not stream audio")
sys.exit(-1)
print text
# Play the audio using the platform's default player
if sys.platform == "win32":
os.startfile(output)
else:
# the following works on Mac and Linux. (Darwin = mac, xdg-open = linux).
opener = "open" if sys.platform == "darwin" else "xdg-open"
subprocess.call([opener, output])
def openGate(accName, carParkName, bookNum):
session = Session(region_name="us-east-1")
polly = session.client('polly')
s3 = boto3.client('s3')
filename = "voice/", bookNum,".mp3"
filename = str(filename)
text = accName + \
". Welcome to " + \
carParkName + \
". The gate will now open. Enjoy your last day at S.A. Launch."
print (text)
#try:
response = polly.synthesize_speech(
Text=str(text),
OutputFormat="mp3",
VoiceId="Joanna")
with closing(response["AudioStream"]) as stream:
s3.put_object(ACL='public-read', Bucket='sa-launch-demo-onedimsum', Key='voice/filename.mp3', Body=stream.read())
class Client_S3:
def __init__(self, iamProfilename):
from boto3 import Session
self.session = Session(profile_name=iamProfilename)
self.s3 = self.session.client('s3')
def _getBucketList(self):
# Reference : http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.list_buckets
response = self.s3.list_buckets()
return(response)
def _removeSpecificBucket(self, bucketName):
# Reference : http://boto3.readthedocs.io/en/latest/reference/services/s3.html#S3.Client.delete_bucket
response = self.s3.delete_bucket(
Bucket= bucketName
)
return(response)
def _render_with_polly(self, blocks, outfile, user):
from boto3 import Session
from botocore.exceptions import BotoCoreError, ClientError
session = Session(profile_name=user)
polly = session.client("polly")
nblocks = len(blocks)
aud = b''
for i, ssml in enumerate(blocks):
print('{0}/{1}\r'.format(i, nblocks), end='')
sys.stdout.flush()
response = polly.synthesize_speech(Text=ssml,
TextType='ssml',
VoiceId='Matthew',
OutputFormat='mp3')
aud += response["AudioStream"].read()
with open(outfile, 'bw') as f:
f.write(aud)
print('Done... ' + outfile)
def do_upload_file(archive_file, upload_location):
"""
:param archive_file:
:param upload_location:
:return:
"""
session = Session()
client = session.client(
"s3",
region_name="nyc3",
endpoint_url="https://nyc3.digitaloceanspaces.com",
aws_access_key_id=DO_ACCESS_KEY_ID,
aws_secret_access_key=DO_SECRET_ACCESS_KEY,
)
client.upload_file(archive_file, DO_SPACE, upload_location)
def initiate_ta_refresh(account_id):
"""
Assumes the specified role in account and initiates the TA refresh
"""
# beginning the assume role process for account
sts_client = client('sts')
response = sts_client.assume_role(RoleArn='arn:aws:iam::' + account_id +
':role/' + environ['CheckRoleName'],
RoleSessionName='AWSLimits')
# storing STS credentials
session = Session(
aws_access_key_id=response['Credentials']['AccessKeyId'],
aws_secret_access_key=response['Credentials']['SecretAccessKey'],
aws_session_token=response['Credentials']['SessionToken'],
region_name='us-east-1')
print "Assumed session for " + account_id + ". Initiating TA Report refresh..."
# now we create a TA client
support_client = session.client('support', region_name='us-east-1')
# requesting a new check for TA
try:
support_client.refresh_trusted_advisor_check(checkId='eW7HH0l7J9')
except botocore.exceptions.ClientError as e:
if e.response['Error']['Code'] == "SubscriptionRequiredException":
print "This account doesn't have AWS Premium Support and can't access the support API. Continuing with other limit checks"
payload = {}
payload['AccountId'] = account_id
payload['Skip_TA'] = True
payloadbytes = dumps(payload)
# now we invoke the limit check lambda for the remaining services
lambda_client = client('lambda', region_name=environ['Region'])
lambda_client.invoke(FunctionName=environ['RetrieveResultsLambda'],
InvocationType='Event',
Payload=payloadbytes)
# no need to continue and create the cloudwatch event since TA check will never complete
# since it was never started
return account_id
else:
print(e)
print "Initiated TA Report refresh"
# create our CWE which will kick off the other lambda to retrieve
# the report and check limits
create_cwe(session, str(account_id))
def deploy_to_routing_server(
session: boto3.Session,
instance_id: str,
key_pair: str,
push_directory: str,
) -> bool:
client = session.client("ec2")
server_host, allocation_id, association_id = get_instance_address(
session, instance_id)
keypair_file = os.path.join(DEFAULT_KEY_PAIR_DIRECTORY, f"{key_pair}.pem")
print("Uploading files to server, then attempting to run")
try:
remote_server = f"{AMI_DEFAULT_USER}@{server_host}"
dest = f"{remote_server}:/home/ec2-user/"
try_server_push([
"scp",
"-o",
"StrictHostKeyChecking=no",
"-i",
keypair_file,
"-r",
f"{push_directory}",
dest,
])
subprocess.check_call(
[
"ssh",
"-i",
keypair_file,
remote_server,
"bash",
"/home/ec2-user/routing_server/setup/init_server.sh",
],
env=dict(os.environ, SSH_AUTH_SOCK=""),
)
detete_instance_address(session, allocation_id, association_id)
print("Server setup complete!")
except Exception as e:
detete_instance_address(session, allocation_id, association_id)
raise e
return True
def create_s3_session(url, connection={}):
url = url_util.parse(url)
if url.scheme != 's3':
raise ValueError(
'Can not create S3 session from URL with scheme: {SCHEME}'.format(
SCHEME=url.scheme))
# NOTE(opadron): import boto and friends as late as possible. We don't
# want to require boto as a dependency unless the user actually wants to
# access S3 mirrors.
from boto3 import Session
from botocore.exceptions import ClientError
s3_connection = {}
if connection:
if connection['access_token']:
s3_connection["aws_session_token"] = connection["access_token"]
if connection["access_pair"][0]:
s3_connection["aws_access_key_id"] = connection["access_pair"][0]
s3_connection["aws_secret_access_key"] = connection["access_pair"][
1]
if connection["profile"]:
s3_connection["profile_name"] = connection["profile"]
session = Session(**s3_connection)
s3_client_args = {"use_ssl": spack.config.get('config:verify_ssl')}
endpoint_url = os.environ.get('S3_ENDPOINT_URL')
if endpoint_url:
s3_client_args['endpoint_url'] = _parse_s3_endpoint_url(endpoint_url)
elif connection and 'endpoint_url' in connection:
s3_client_args["endpoint_url"] = _parse_s3_endpoint_url(
connection["endpoint_url"]) # noqa: E501
# if no access credentials provided above, then access anonymously
if not session.get_credentials():
from botocore import UNSIGNED
from botocore.client import Config
s3_client_args["config"] = Config(signature_version=UNSIGNED)
client = session.client('s3', **s3_client_args)
client.ClientError = ClientError
return client
def _get_polly_client(self,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_session_token=None,
region_name=None,
with_service_model_patch=False):
# Note we get a new botocore session each time this function is called
# to avoid potential problems caused by inner state of the session.
botocore_session = get_session()
if with_service_model_patch:
# Older versions of botocore don't have polly. We can possibly fix it by appending
# extra path with polly service model files to the search path.
current_dir = os.path.dirname(os.path.abspath(__file__))
service_model_path = os.path.join(current_dir, 'data', 'models')
botocore_session.set_config_variable('data_path',
service_model_path)
self.loginfo('patching service model data path: {}'.format(
service_model_path))
botocore_session.user_agent_extra = self._generate_user_agent_suffix()
session = Session(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
region_name=region_name,
botocore_session=botocore_session)
try:
return session.client("polly")
except UnknownServiceError:
# the first time we reach here, we try to fix the problem
if not with_service_model_patch:
return self._get_polly_client(aws_access_key_id,
aws_secret_access_key,
aws_session_token,
region_name,
with_service_model_patch=True)
else:
# we have tried our best, time to panic
self.logerr(
'Amazon Polly is not available. Please install the latest boto3.'
)
raise
def handler_factory(session: boto3.Session,
cost_center: str) -> Callable[[dict, object], dict]:
assert cost_center, "expecting cost center ID"
lambda_client = session.client("lambda")
def handler(event: dict, context: object) -> dict:
assert "arn" in event, "expecting 'arn' key"
lambda_client.tag_resource(
Resource=event["arn"],
Tags={"CostCenter": cost_center},
)
response = lambda_client.list_tags(Resource=event["arn"], )
return response["Tags"]
return handler
def display_access_keys(session: boto3.Session):
target_key = session.get_credentials().access_key
iam = session.client('iam')
response = iam.list_access_keys()['AccessKeyMetadata']
safe_print('Displaying Access Keys'.center(45, '='))
safe_print('ACCESS KEY ID'.ljust(20), end=' ')
safe_print('STATUS'.ljust(10), end=' ')
safe_print('CREATE DATE'.ljust(10))
for key_metadata in response:
color = None if target_key != key_metadata.get(
'AccessKeyId') else colorama.Fore.MAGENTA
safe_print(key_metadata.get('AccessKeyId'), end=' ', color=color)
safe_print('[{}]'.format(key_metadata.get('Status')).ljust(10),
end=' ',
color=color)
safe_print('{}'.format(
key_metadata.get('CreateDate').strftime('%d/%M/%Y')).ljust(10),
color=color)
safe_print(''.center(45, '='))
def lookup_user_by_name(session: boto3.Session,
ids: Ids,
user_name,
*,
cache=None):
if cache is None:
cache = {}
cache_key_name = f"{_CACHE_KEY_PREFIX_USER_NAME}{user_name}"
if cache_key_name in cache:
LOGGER.debug(f"Found user {user_name} in cache")
user = cache[cache_key_name]
if isinstance(user, LookupError):
raise user
return user
LOGGER.debug(f"Looking up user {user_name}")
identity_store = session.client('identitystore')
filters = [{'AttributePath': 'UserName', 'AttributeValue': user_name}]
response = identity_store.list_users(IdentityStoreId=ids.identity_store_id,
Filters=filters)
if len(response['Users']) == 0:
err = LookupError("No user named {} found".format(user_name))
cache[cache_key_name] = err
raise err
elif len(response['Users']) > 1:
err = LookupError("{} users named {} found".format(
len(response['Users']), user_name))
cache[cache_key_name] = err
raise err
user = response['Users'][0]
user_id = user["UserId"]
cache_key_id = f"{_CACHE_KEY_PREFIX_USER_ID}{user_id}"
cache[cache_key_id] = user
cache[cache_key_name] = user
return user
def get_session(self):
if self.profile_name:
self._log.info(
'using AWS credential profile %s', self.profile_name)
try:
kwargs = {'profile_name': self.profile_name}
if self.region:
kwargs['region_name'] = self.region
session = Session(**kwargs)
except Exception as ex:
self._log.fatal(
'Could not connect to AWS using profile %s: %s',
self.profile_name, ex)
raise
else:
self._log.debug(
'getting an AWS session with the default provider')
kwargs = {}
if self.region:
kwargs['region_name'] = self.region
session = Session(**kwargs)
if self.role_arn:
self._log.info(
'attempting to assume STS self.role %s', self.role_arn)
try:
self.role_creds = session.client('sts').assume_role(
RoleArn=self.role_arn,
RoleSessionName='repoman-%s' % time.time(),
DurationSeconds=3600)['Credentials']
except Exception as ex:
self._log.fatal(
'Could not assume self.role %s: %s',
self.role_arn, ex)
raise
kwargs = {
'aws_access_key_id': self.role_creds['AccessKeyId'],
'aws_secret_access_key': self.role_creds['SecretAccessKey'],
'aws_session_token': self.role_creds['SessionToken']}
if self.region:
kwargs['region_name'] = self.region
session = Session(**kwargs)
return session
class AutoRefreshableSession:
METHOD = "iam-role"
DEFAULT_RETRIES = 5
DEFAULT_METADATA_SERVICE_TIMEOUT = 10 # secs
def __init__(self,
retries=DEFAULT_RETRIES,
metadata_service_timeout=DEFAULT_METADATA_SERVICE_TIMEOUT):
self.instance_metadata_fetcher = InstanceMetadataFetcher()
self.instance_metadata_fetcher._num_attempts = retries
self.instance_metadata_fetcher._timeout = metadata_service_timeout
self.instance_metadata_fetcher._needs_retry_for_credentials = self.needs_retry_for_credentials
self.instance_metadata_provider = InstanceMetadataProvider(
self.instance_metadata_fetcher)
def check_for_missing_keys(self, required_cred_fields, response):
print(response.content)
credentials = json.loads(response.content)
for field in required_cred_fields:
if field not in credentials:
print('Retrieved credentials is missing required field: %s',
field)
return True
return False
def needs_retry_for_credentials(self, response):
return (self.instance_metadata_fetcher._is_non_ok_response(response)
or self.instance_metadata_fetcher._is_empty(response)
or self.instance_metadata_fetcher._is_invalid_json(response)
or self.check_for_missing_keys(
self.instance_metadata_fetcher._REQUIRED_CREDENTIAL_FIELDS,
response))
def _get(self, region):
self.session = get_session()
self.session._credentials = self.instance_metadata_provider.load()
self.session.set_config_variable("region", region)
self.autorefresh_session = Session(botocore_session=self.session)
return self
def client(self, service_name):
return self.autorefresh_session.client(service_name=service_name)
def postscript():
try:
body = json.load(request.body)
except:
raise ValueError
script = body["script"]
ssml = body["ssml"]
if ssml == "true":
tt = "ssml"
else:
tt = "text"
# 現在時刻の取得
nowtime = datetime.datetime.now()
nowtime = nowtime + datetime.timedelta(hours=9)
nowtime = nowtime.strftime("%Y%m%d%H%M%S")
session = Session(region_name="ap-northeast-1")
client = session.client("polly")
filename = "/home/ec2-user/demotenki/static/polly/" + nowtime + ".mp3"
response = client.synthesize_speech(Text=script,
TextType=tt,
OutputFormat="mp3",
VoiceId="Takumi")
if "AudioStream" in response:
with closing(response["AudioStream"]) as stream:
output = filename
with open(output, "wb") as file:
file.write(stream.read())
filepath = "http://13.113.245.130/file/polly/" + nowtime + ".mp3"
result = {"filepath": filepath, "script": script, "time": nowtime}
# JSONにエンコードして返す。
# return json.dumps(body)
return json.dumps(result)
def device_list(self,
session: boto3.Session = None,
identity_id: str = None):
if (session == None):
session = self.get_session()
identity_id = self.aws_identity_id
client = session.client('lambda')
resp = client.invoke(FunctionName='Device_Manager_V2',
InvocationType="RequestResponse",
Payload=json.dumps({
"Device_Manager_Request":
"query",
"Identity_Id":
self.aws_identity_id,
"Region_Info":
session.region_name
}))
payload = json.loads(resp['Payload'].read())
return payload['Request_Cotent']
def upload_s3(file_name, bucket, object_name=None):
"""Upload a file to an S3 bucket
:param file_name: File to upload
:param bucket: Bucket to upload to
:param object_name: S3 object name. If not specified then file_name is used
:return: True if file was uploaded, else False
"""
# If S3 object_name was not specified, use file_name
if object_name is None:
object_name = file_name
# Upload the file
session = Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
s3_client = session.client('s3')
try:
response = s3_client.upload_file(file_name, bucket, object_name, ExtraArgs={'ACL': 'public-read'})
except ClientError as e:
return False
return True
def delete_cluster(session: boto3.Session, config: Config,
args: argparse.Namespace) -> None:
"""
Deletes a redshift cluster.
This function tries to take args.cluster_id as the cluster identifier for deletion. If the command is run without
the --cluster-id option, then it deletes the cluster identified as `config.cluster_identifier`
:param session: a boto3.Session instance
:param config: Config instance containing cluster configuration information
:param args: argparse.Namespace object containing a `cluster_id` field; this parameter
"""
cluster_id = config.cluster_identifier if args.cluster_id is None else args.cluster_id
print(f'Deleting cluster {colorify(cluster_id, "warning")}...')
redshift = session.client('redshift')
response = redshift.delete_cluster(ClusterIdentifier=cluster_id,
SkipFinalClusterSnapshot=True)
# Wait for cluster to be deleted
redshift.get_waiter('cluster_deleted').wait(ClusterIdentifier=cluster_id)
print(colorify('Cluster successfully deleted.', "warning"))
class S3(object):
def __init__(self, bucket):
self.config = get_config()
self.aws_config = self.__aws_config()
self.session = Session(self.aws_access_key_id,
self.aws_secret_access_key)
self.bucket = bucket
self.s3 = self.session.client('s3')
def __aws_config(self):
path = os.path.expanduser('~/.aws/credentials')
if os.path.exists(path):
config = ConfigParser()
config.readfp(open(path))
return config
@property
def aws_access_key_id(self):
config = self.config.get('aws', 'aws_access_key_id')
if config:
return config
aws = self.aws_config.get('default', 'aws_access_key_id')
if aws:
return aws
raise Exception('Missing aws_access_key_id from config locations')
@property
def aws_secret_access_key(self):
config = self.config.get('aws', 'aws_secret_access_key')
if config:
return config
aws = self.aws_config.get('default', 'aws_secret_access_key')
if aws:
return aws
raise Exception('Missing aws_secret_access_key from config locations')
def get_session(self):
if self.profile_name:
self._log.info('using AWS credential profile %s',
self.profile_name)
try:
kwargs = {'profile_name': self.profile_name}
if self.region:
kwargs['region_name'] = self.region
session = Session(**kwargs)
except Exception as ex:
self._log.fatal(
'Could not connect to AWS using profile %s: %s',
self.profile_name, ex)
raise
else:
self._log.debug('getting an AWS session with the default provider')
kwargs = {}
if self.region:
kwargs['region_name'] = self.region
session = Session(**kwargs)
if self.role_arn:
self._log.info('attempting to assume STS self.role %s',
self.role_arn)
try:
self.role_creds = session.client('sts').assume_role(
RoleArn=self.role_arn,
RoleSessionName='repoman-%s' % time.time(),
DurationSeconds=3600)['Credentials']
except Exception as ex:
self._log.fatal('Could not assume self.role %s: %s',
self.role_arn, ex)
raise
kwargs = {
'aws_access_key_id': self.role_creds['AccessKeyId'],
'aws_secret_access_key': self.role_creds['SecretAccessKey'],
'aws_session_token': self.role_creds['SessionToken']
}
if self.region:
kwargs['region_name'] = self.region
session = Session(**kwargs)
return session
def get_dns_records(record, zone_name):
"""
Returns only the records of given cluster record.
Args:
record (dns name), zone_name (route 53 zone)
Returns:
boolean (true or false), record domain
"""
session = Session()
route53 = session.client('route53')
zone_id = get_dns_hostedid(zone_name)
record = record + '.' if not record.endswith('.') else record
logger.info("Retrieved Zone ID: ", zone_id)
if zone_id is None:
logger.info(
"Error: Unable to fetch the Hosted zone id of the given zone name: "
+ zone_name)
return None, {
"Description":
"Unable to fetch the details of the given zone : " + zone_name
}
try:
response = route53.list_resource_record_sets(HostedZoneId=zone_id,
StartRecordName=zone_name)
record_domain = [
rec for rec in response.get('ResourceRecordSets')
if record == rec.get('Name')
]
except Exception as e:
logger.error("Error: Unable to fetch the details of the given zone " +
zone_id)
logger.exception(e)
raise botocore.exceptions.ClientError(e, 'get_dns_records')
return True, record_domain
def create_target_group(
session: boto3.Session,
vpc_id: str,
instance_id: str,
group_name="mephisto-fallback",
) -> str:
"""
Create a target group for the given instance
"""
client = session.client("elbv2")
group_name_hash = hashlib.md5(group_name.encode("utf-8")).hexdigest()
anti_collision_group_name = f"{group_name_hash[:8]}-{group_name}"
final_group_name = f"{anti_collision_group_name[:28]}-tg"
create_target_response = client.create_target_group(
Name=final_group_name[:32],
Protocol="HTTP",
ProtocolVersion="HTTP1",
Port=5000,
VpcId=vpc_id,
Matcher={
"HttpCode": "200-299",
},
TargetType="instance",
Tags=[
{
"Key": "string",
"Value": "string"
},
],
)
target_group_arn = create_target_response["TargetGroups"][0][
"TargetGroupArn"]
client.register_targets(
TargetGroupArn=target_group_arn,
Targets=[{
"Id": instance_id,
}],
)
return target_group_arn
def get_certificate(session: boto3.Session,
domain_name: str) -> Dict[str, str]:
"""
Gets the certificate for the given domain name, and returns
the dns validation name and target and cert arn ('Name' and 'Value', 'arn')
"""
client = session.client("acm")
cert_domain_name = f"*.{domain_name}"
certificate_arn = find_certificate_arn(session, cert_domain_name)
if certificate_arn is None: # cert not yet issued
logger.debug("Requesting new certificate")
response = client.request_certificate(
DomainName=cert_domain_name,
ValidationMethod="DNS",
IdempotencyToken=f"{domain_name.split('.')[0]}request",
Options={
"CertificateTransparencyLoggingPreference": "ENABLED",
},
)
certificate_arn = response["CertificateArn"]
else:
logger.debug(f"Using existing certificate {certificate_arn}")
attempts = 0
sleep_time = 2
details = None
while attempts < MAX_RETRIES:
try:
details = client.describe_certificate(
CertificateArn=certificate_arn, )
return_data = details["Certificate"]["DomainValidationOptions"][0][
"ResourceRecord"]
return_data["arn"] = certificate_arn
return return_data
except KeyError:
# Resource record not created yet, try again
attempts += 1
logger.info(
f"Attempt {attempts} had no certification details, retrying")
time.sleep(sleep_time)
sleep_time *= 2
raise Exception("Exceeded MAX_RETRIES waiting for certificate records")
def deliver_sns(self, topic, subject, msg, data):
account = topic.split(':')[4]
if account in self.sns_cache:
sns = self.sns_cache[account]
else:
if account not in self.config['cross_accounts']:
log.error("No cross account role for sending sns to %s" %
topic)
return
creds = self.sts.assume_role(
RoleArn=self.config['cross_accounts'][account],
RoleSessionName="CustodianNotification")['Credentials']
session = Session(aws_access_key_id=creds['AccessKeyId'],
aws_secret_access_key=creds['SecretAccessKey'],
aws_session_token=creds['SessionToken'])
self.sns_cache[account] = sns = session.client('sns')
log.info("Sending account:%s policy:%s sns:%s to %s" %
(data.get('account', ''), data['policy']['name'],
data['action'].get('template', 'default'), topic))
sns.publish(TopicArn=topic, Subject=subject, Message=msg)
def set_configuration_ssm_parameters(management_session: boto3.Session,
params: dict) -> None:
"""Set Configuration SSM Parameters.
Args:
management_session: Management account session
params: Parameters
"""
ssm_client: SSMClient = management_session.client("ssm")
ssm_parameter_value = {
"ENABLE_BLOCK_PUBLIC_ACLS": params["ENABLE_BLOCK_PUBLIC_ACLS"],
"ENABLE_IGNORE_PUBLIC_ACLS": params["ENABLE_IGNORE_PUBLIC_ACLS"],
"ENABLE_BLOCK_PUBLIC_POLICY": params["ENABLE_BLOCK_PUBLIC_POLICY"],
"ENABLE_RESTRICT_PUBLIC_BUCKETS":
params["ENABLE_RESTRICT_PUBLIC_BUCKETS"],
"ROLE_SESSION_NAME": params["ROLE_SESSION_NAME"],
"ROLE_TO_ASSUME": params["ROLE_TO_ASSUME"],
}
put_ssm_parameter(ssm_client, f"{SSM_PARAMETER_PREFIX}", "",
json.dumps(ssm_parameter_value))
def test_select_projected_filtered_topk_with_cursor():
cfg = Config(region_name="us-east-1", parameter_validation=False, max_pool_connections=10,
s3={'payload_signing_enabled': False})
session = Session()
s3 = session.client('s3', use_ssl=False, verify=False, config=cfg)
num_rows = 0
cur = PandasCursor(s3) \
.parquet() \
.select('parquet/small.multicolumn.9999.parquet', 'select b from s3Object where a > 5000 limit 100')
try:
dfs = cur.execute()
for df in dfs:
num_rows += len(df)
print("{}:{}".format(num_rows, df))
assert num_rows == 100
finally:
cur.close()
def _call_service_method(self, metric):
session_credentials = RefreshableCredentials.create_from_metadata(
metadata=self._refresh(),
refresh_using=self._refresh,
method="sts-assume-role",
)
session = get_session()
session._credentials = session_credentials
autorefresh_session = Session(botocore_session=session)
service = autorefresh_session.client(metric.service)
# service = self._session.client(metric.service)
service_method = getattr(service, metric.method)
next_token = ''
responses = []
kwargs = dict(**metric.method_args)
while next_token is not None:
response = service_method(**kwargs)
next_token = response.get('NextToken', None)
responses += jmespath.search(metric.search, response)
kwargs["NextToken"] = next_token
return responses
def create_load_balancer(
session: boto3.Session,
subnet_ids: List[str],
security_group_id: str,
vpc_id: str,
) -> str:
"""
Creates a load balancer and returns the balancer's arn
"""
client = session.client("elbv2")
create_response = client.create_load_balancer(
Name="mephisto-hosts-balancer",
Subnets=subnet_ids,
SecurityGroups=[security_group_id],
Scheme="internet-facing",
Type="application",
IpAddressType="ipv4",
)
balancer_arn = create_response["LoadBalancers"][0]["LoadBalancerArn"]
return balancer_arn
def test_write_parquet_file_to_s3():
test_path = tests_path + "/{}".format(test_write_parquet_file_to_s3.__name__)
ensure_dir(test_path)
upload_parquet_file = "{}/supplier.upload.parquet".format(test_path)
download_parquet_file = "{}/supplier.download.parquet".format(test_path)
convert_csv_file_to_parquet_file('{}/tests/parquet/supplier.csv'.format(ROOT_DIR), upload_parquet_file)
cfg = Config(region_name="us-east-1", parameter_validation=False, max_pool_connections=10)
session = Session()
s3 = session.client('s3', config=cfg)
s3.upload_file(upload_parquet_file, 's3filter', 'parquet/supplier.parquet')
s3.download_file('s3filter', 'parquet/supplier.parquet', download_parquet_file)
table = pq.read_table(download_parquet_file)
df = arrow_to_pandas(table)
assert_supplier_table(df, 10000, use_ordinal_columns=False)
def _create_cluster(config: Config, role_arn: str,
session: boto3.Session) -> str:
"""
Creates a redshift cluster and returns its endpoint.
:param config: a Config instnce
:param role_arn: a IAM Role ARN string
:param session: a boto3.Session instance
:return endpoint: the endpoint for the redshift cluster
"""
print('Creating cluster...')
redshift = session.client('redshift')
response = redshift.create_cluster(
# Cluster
ClusterType=config.cluster_type,
NodeType=config.node_type,
NumberOfNodes=config.num_nodes,
# Identifiers & Credentials
DBName=config.db_name,
ClusterIdentifier=config.cluster_identifier,
MasterUsername=config.db_user,
MasterUserPassword=config.db_password,
# Roles (for s3 access) and security group
IamRoles=[role_arn],
VpcSecurityGroupIds=[config.security_group_id])
# Wait for cluster to be available
print(f'Waiting for cluster to become available...')
redshift.get_waiter('cluster_available').wait(
ClusterIdentifier=config.cluster_identifier)
# Display cluster properties
cluster_props = redshift.describe_clusters(
ClusterIdentifier=config.cluster_identifier)['Clusters'][0]
endpoint = cluster_props['Endpoint']['Address']
print(
f'Cluster {colorify(config.cluster_identifier, "bold", "okgreen")} successfully created and available.'
)
print(
f' Cluster endpoint: {colorify(endpoint, "bold", "okblue", "underline")}'
)
return endpoint
def deliver_sns(self, topic, subject, msg, data):
account = topic.split(':')[4]
if account in self.sns_cache:
sns = self.sns_cache[account]
else:
if account not in self.config['cross_accounts']:
log.error(
"No cross account role for sending sns to %s" % topic)
return
creds = self.sts.assume_role(
RoleArn=self.config['cross_accounts'][account],
RoleSessionName="CustodianNotification")['Credentials']
session = Session(
aws_access_key_id=creds['AccessKeyId'],
aws_secret_access_key=creds['SecretAccessKey'],
aws_session_token=creds['SessionToken'])
self.sns_cache[account] = sns = session.client('sns')
log.info("Sending account:%s policy:%s sns:%s to %s" % (
data.get('account', ''),
data['policy']['name'],
data['action'].get('template', 'default'),
topic))
sns.publish(TopicArn=topic, Subject=subject, Message=msg)
def __init__(self, access_key, secret_key, region='us-east-1'):
session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
self.client = session.client('ecs')
aws_secret_access_key=aws_secret_access_key,
region_name=region_name)
tmp_dir = tempfile.mkdtemp()
dst_zip_path = os.path.join(tempfile.gettempdir(), "%s.zip" % lambda_function_name)
print("[x] Using:\n\t- %s\n\t- %s" % (code_path, site_packages_dir))
try:
copy_files(src=code_path, dst=tmp_dir)
copy_files(src=site_packages_dir, dst=tmp_dir)
if os.path.exists(dst_zip_path):
os.remove(dst_zip_path)
zip_dir(tmp_dir, dst_zip_path)
finally:
shutil.rmtree(tmp_dir)
print("[x] Sending zip to s3...")
s3 = aws_session.resource('s3')
s3_file_name = '%s.zip' % lambda_function_name
s3.Bucket(s3_bucket_name).put_object(Key=s3_file_name, Body=open(dst_zip_path, 'rb'))
print("[x] Updating lambda code...")
lambda_ = aws_session.client('lambda')
lambda_.update_function_code(FunctionName=lambda_function_name,
S3Bucket=s3_bucket_name,
S3Key=s3_file_name,
Publish=True)
print("[x] Done")
from __future__ import print_function
import json
from HTMLParser import HTMLParser
import requests
from boto3 import Session
s=Session(region_name='eu-west-1')
awslambda = s.client('lambda')
idlist=[]
def lambda_handler(event, context):
class MyHTMLParser(HTMLParser):
def handle_starttag(self, tag, attrs):
for attr in attrs:
if "data-ds-appid" in attr:
game_id=attr[1]
if game_id not in idlist:
idlist.append(game_id)
return(idlist)
#Find the early access game pagination count
url='http://store.steampowered.com/search/tabpaginated/render/?query=&start=10&count=1&genre=70&tab=NewReleases&cc=TR&l=english'
r=requests.get(url)
data = json.loads(r.text)
total_count=data['total_count']
pagination=total_count/10+1
#Create the list
i=0
while(i<pagination*10):
url='http://store.steampowered.com/search/tabpaginated/render/?query=&start='+str(i)+'&count=10+&genre=70&tab=NewReleases&cc=TR&l=english'
class VpcHandler(base.BaseHandler):
"""Manage ``EC2.Vpc`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra
# features to the base class EC2.Vpc
event = 'creating-resource-class.ec2.Vpc'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a VPC, or update it if it already exists."""
self._resource = self._load(self._name, manager_name='vpcs')
if self._resource:
logger.info('VPC %(name)s|%(cidr_block)s already exists.', self._ctx)
else:
self._create_vpc()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a VPC object."""
self._resource = self._load(self._name, manager_name='vpcs')
if self._resource:
self._update_vpc()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('VPC %s does not exist.', self._name)
def delete_resource(self):
"""Delete a ``Vpc`` object."""
self._resource = self._load(self._name, manager_name='vpcs')
if self._resource:
# If an internet gateway is attached to the VPC,
# we try to detach it first
filters = [{
'Name': 'attachment.vpc-id',
'Values': [self._resource.id]
}]
items = list(self._resource.internet_gateways.filter(Filters=filters))
if items:
igw = items[0]
try:
self._resource.detach_internet_gateway(InternetGatewayId=igw.id)
except ClientError as exc:
logger.error(exc)
# Delete the VPC
logger.info('Removing VPC %(name)s|%(cidr_block)s.', self._ctx)
try:
self._client.delete_vpc(VpcId=self._resource.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error('VPC %s does not exist.', self._name)
def _create_vpc(self):
"""Create a ``Vpc`` object."""
logger.info('Creating VPC %(name)s|%(cidr_block)s.', self._ctx)
try:
self._resource = self._ec2.create_vpc(CidrBlock=self._ctx['cidr_block'])
except ClientError as exc:
logger.error(exc)
return
# Wait for the new VPC to become
# available before going any further
waiter = self._client.get_waiter('vpc_available')
waiter.wait(VpcIds=[self._resource.id])
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_vpc()
def _update_vpc(self):
"""Update a ``Vpc`` object."""
self._update_tags()
self._manage_dhcp_options_set()
self._manage_internet_gateway()
def _manage_dhcp_options_set(self):
"""Manage the DHCP options set association."""
dhcp_name = self._ctx.get('dhcp_options')
if not dhcp_name:
return
# Try to load the ``EC2.DhcpOptions`` object
dhcp = self._load(dhcp_name, manager_name='dhcp_options_sets')
if dhcp:
if dhcp.id != self._resource.dhcp_options_id:
# Associate DHCP options set
logger.info(
'Associating DHCP options set %s with VPC %s.', dhcp_name, self._name
)
try:
self._resource.associate_dhcp_options(DhcpOptionsId=dhcp.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error(
'Unable to associate DHCP options set %s with VPC %s'
' because it does not exist.', dhcp_name, self._name
)
def _manage_internet_gateway(self):
"""Manage the internet gateway attachment."""
igw_name = self._ctx.get('internet_gateway')
filters = [{'Name': 'attachment.vpc-id', 'Values': [self._resource.id]}]
internet_gateways = list(self._ec2.internet_gateways.filter(Filters=filters))
if not igw_name:
if internet_gateways:
# If an internet gateway is attached to the VPC but not defined in
# the configuration, we don't take the responsibility to detach it
logger.warning(
'An internet gateway is still attached to VPC %s, although there'
' is no corresponding element in the configuration.', self._name
)
return
# Try to load the ``EC2.InternetGateway`` object
igw = self._load(igw_name, manager_name='internet_gateways')
if igw:
if igw.attachments:
vpc_id = igw.attachments[0]['VpcId']
if vpc_id != self._resource.id:
# The internet gateway is already attached to another VPC
logger.warning(
'Unable to attach internet gateway %s to VPC %s because it '
'is already attached to another VPC.', igw_name, self._name
)
elif internet_gateways:
logger.warning(
'Unable to attach internet gateway %s to VPC %s because another internet'
' gateway is already attached to the latter.', igw_name, self._name
)
else:
logger.info(
'Attaching internet gateway %s to VPC %s.', igw_name, self._name
)
try:
self._resource.attach_internet_gateway(InternetGatewayId=igw.id)
# Update cache to reflect new internet gateway status. If
# there is another attempt to attach this gateway during
# the same program execution, a warning will be raised.
igw.reload()
with self._lock:
self._cache[igw_name] = igw
except ClientError as exc:
logger.error(exc)
else:
logger.error(
'Unable to attach internet gateway %s to VPC %s '
'because it does not exist.', igw_name, self._name
)
class SecurityGroupHandler(BaseHandler):
"""Manage ``EC2.SecurityGroup`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra
# features to the base class EC2.SecurityGroup
event = 'creating-resource-class.ec2.SecurityGroup'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a security group, or update it if it already exists."""
self._resource = self._load(self._name, manager_name='security_groups')
if self._resource:
logger.info('Security group %s already exists.', self._name)
else:
self._create_security_group()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a security group object."""
self._resource = self._load(self._name, manager_name='security_groups')
if self._resource:
self._update_security_group()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('Security group %s does not exist.', self._name)
@retry(**retry_params)
def delete_resource(self):
"""Delete a ``SecurityGroup`` object."""
self._resource = self._load(self._name, manager_name='security_groups')
if self._resource:
logger.info('Removing security group %s.', self._name)
try:
self._resource.delete(GroupId=self._resource.id)
except ClientError as exc:
# This is intended to prevent potential errors when trying
# to remove security groups that may depend on each other.
# If a 'DependencyViolation' is raised, we try to delete
# the security group again after a short delay.
if exc.response['Error']['Code'] == 'DependencyViolation':
logger.info(
'Security group %s has a dependency and can '
'not be removed. Trying again...', self._name
)
raise DependencyViolationError
else:
logger.error(exc)
else:
logger.error('Security group %s does not exist.', self._name)
def _create_security_group(self):
"""Create a ``SecurityGroup`` object."""
vpc_name = self._ctx['vpc']
vpc = self._load(vpc_name, manager_name='vpcs')
if vpc:
logger.info('Creating security group %s.', self._name)
try:
self._resource = self._ec2.create_security_group(
GroupName=self._name,
Description=self._ctx['description'],
VpcId=vpc.id
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
return
# Remove the default outbound rule authorizing all outgoing traffic
ip_permission = {
'IpProtocol': '-1',
'FromPort': -1,
'ToPort': -1,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}],
'UserIdGroupPairs': []
}
logger.info('%s: revoking default outbound rule.', self._name)
try:
self._resource.revoke_egress(IpPermissions=[ip_permission])
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_security_group()
else:
logger.error(
'Unable to create security group %s because VPC %s does not'
' exist. Operation not performed.', self._name, vpc_name
)
def _update_security_group(self):
"""Update a ``SecurityGroup`` object."""
self._update_tags()
self._update_ingress_rules()
self._update_egress_rules()
def _update_ingress_rules(self):
"""Manage security group ingress rules."""
current_ingress_rules = self._resource.ingress_rules
new_ingress_rules = []
to_authorize = []
to_revoke = []
for rule in self._ctx.get('ingress', []):
for source in rule['sources']:
ip_permission = {
'IpProtocol': str(rule['ip_protocol']),
'FromPort': int(rule['from_port']),
'ToPort': int(rule['to_port'])
}
try:
ipaddress.ip_network(source)
# 'source' attribute is a valid CIDR
ip_permission['IpRanges'] = [{'CidrIp': source}]
ip_permission['UserIdGroupPairs'] = []
except ValueError:
# Assuming we got a security group ID
sg = self._load(source, manager_name='security_groups')
if sg:
pair = {'GroupId': sg.id, 'UserId': sg.owner_id}
ip_permission['UserIdGroupPairs'] = [pair]
ip_permission['IpRanges'] = []
else:
logger.error(
"%s: invalid value (source security group "
"'%s' does not exist).", self._name, source
)
continue
new_ingress_rules.append(
RuleWrapper(ip_permission, flow='source')
)
for rule in set(new_ingress_rules) - set(current_ingress_rules):
logger.info('%s: authorizing inbound rule %s.', self._name, rule)
to_authorize.append(rule.ip_permission)
for rule in set(current_ingress_rules) - set(new_ingress_rules):
logger.info('%s: revoking inbound rule %s.', self._name, rule)
to_revoke.append(rule.ip_permission)
try:
if to_authorize:
self._resource.authorize_ingress(IpPermissions=to_authorize)
if to_revoke:
self._resource.revoke_ingress(IpPermissions=to_revoke)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
def _update_egress_rules(self):
"""Manage security group egress rules."""
current_egress_rules = self._resource.egress_rules
new_egress_rules = []
to_authorize = []
to_revoke = []
for rule in self._ctx.get('egress', []):
for destination in rule['destinations']:
ip_permission = {
'IpProtocol': str(rule['ip_protocol']),
'FromPort': int(rule['from_port']),
'ToPort': int(rule['to_port'])
}
try:
ipaddress.ip_network(destination)
# 'destination' attribute is a valid CIDR
ip_permission['IpRanges'] = [{'CidrIp': destination}]
ip_permission['UserIdGroupPairs'] = []
except ValueError:
# Assuming we got a security group ID
sg = self._load(
destination, manager_name='security_groups'
)
if sg:
pair = {'GroupId': sg.id, 'UserId': sg.owner_id}
ip_permission['UserIdGroupPairs'] = [pair]
ip_permission['IpRanges'] = []
else:
logger.error(
"%s: invalid value (destination security group "
"'%s' does not exist).", self._name, destination
)
continue
new_egress_rules.append(
RuleWrapper(ip_permission, flow='destination')
)
for rule in set(new_egress_rules) - set(current_egress_rules):
logger.info('%s: authorizing outbound rule %s.', self._name, rule)
to_authorize.append(rule.ip_permission)
for rule in set(current_egress_rules) - set(new_egress_rules):
logger.info('%s: revoking outbound rule %s.', self._name, rule)
to_revoke.append(rule.ip_permission)
try:
if to_authorize:
self._resource.authorize_egress(IpPermissions=to_authorize)
if to_revoke:
self._resource.revoke_egress(IpPermissions=to_revoke)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
def assume_role(accountID, rgn, event):
ec2_message = ""
cfn_message = ""
rds_message = ""
client = boto3.client('sts')
response = client.assume_role(RoleArn='arn:aws:iam::'+accountID+':role/'+event['CheckRoleName'],
RoleSessionName='AWSLimits')
session = Session(
aws_access_key_id=response['Credentials']['AccessKeyId'],
aws_secret_access_key=response['Credentials']['SecretAccessKey'],
aws_session_token=response['Credentials']['SessionToken'],
region_name=rgn
)
##############
# call trusted advisor for the limit checks
##############
support_client = session.client('support', region_name='us-east-1')
response = support_client.describe_trusted_advisor_check_result(
checkId='eW7HH0l7J9',
language='en'
)
print "Contacting Trusted Advisor..."
# parse the json and find flagged resources that are in warning mode
flag_list = response['result']['flaggedResources']
warn_list=[]
for fr in flag_list:
if fr['metadata'][5] != "Green":
warn_list.append(fr['metadata'][2]+'\n'+'Region: '+fr['metadata'][0]+'\n------------------------'+'\nResource Limit: '+fr['metadata'][3]+'\n'+'Resource Usage: '+fr['metadata'][4]+'\n')
if not warn_list:
print "TA all green"
else:
global ta_message
ta_message = trustedAlert(warn_list)
###############
#call EC2 limits for rgn
###############
ec2_client = session.client('ec2', region_name=rgn)
response = ec2_client.describe_account_attributes()
attribute_list = response['AccountAttributes']
for att in attribute_list:
if att['AttributeName'] == 'max-instances':
limit_of_instances = att['AttributeValues'][0]['AttributeValue']
print"num of limit: "+limit_of_instances
response = ec2_client.describe_instances()
reservation_list = response['Reservations']
num_of_instances = 0
for rsrv in reservation_list:
instance_list = rsrv['Instances']
num_of_instances += len(instance_list)
print "num of instances: "+str(num_of_instances)
#calculate if limit is within threshold
if (float(num_of_instances) / float(limit_of_instances) >= 0.8):
ec2_message = ec2Alert(limit_of_instances, num_of_instances, rgn)
print ec2_message
###############
#cfn resource limit
###############
cfn_client = session.client('cloudformation', region_name=rgn)
stack_limit = cfn_client.describe_account_limits()
if stack_limit['AccountLimits'][0]['Name'] == 'StackLimit':
limit_of_stacks = stack_limit['AccountLimits'][0]['Value']
stacks = cfn_client.describe_stacks()
stack_list = stacks['Stacks']
num_of_stacks = len(stack_list)
if (float(num_of_stacks) / float(limit_of_stacks) >= 0.8):
cfn_message = cloudformationAlert(limit_of_stacks, num_of_stacks, rgn)
print cfn_message
################
#call RDS Limits for rgn
################
rds_client = session.client('rds', region_name=rgn)
instance_limit = rds_client.describe_account_attributes()
service_limit = instance_limit['AccountQuotas'][0]['Max']
service_usage = instance_limit['AccountQuotas'][0]['Used']
if (float(service_usage) / float(service_limit) >= 0.8):
rds_message = rdsAlert(service_limit, service_usage, rgn)
print rds_message
print "Assumed session for "+accountID+" in region "+rgn
rgn_message = ec2_message + cfn_message + rds_message
return rgn_message;
class SubnetHandler(base.BaseHandler):
"""Manage ``EC2.Subnet`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra
# features to the base class EC2.Subnet
event = 'creating-resource-class.ec2.Subnet'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a subnet, or update it if it already exists."""
self._resource = self._load(self._name, manager_name='subnets')
if self._resource:
logger.info('Subnet %s already exists.', self._name)
else:
self._create_subnet()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a subnet object."""
self._resource = self._load(self._name, manager_name='subnets')
if self._resource:
self._update_subnet()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('Subnet %s does not exist.', self._name)
def delete_resource(self):
"""Delete a ``Subnet`` object."""
self._resource = self._load(self._name, manager_name='subnets')
if self._resource:
logger.info('Removing subnet %s.', self._name)
try:
self._client.delete_subnet(SubnetId=self._resource.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error('Subnet %s does not exist.', self._name)
def _create_subnet(self):
"""Create a ``Subnet`` object."""
vpc_name = self._ctx['vpc']
vpc = self._load(vpc_name, manager_name='vpcs')
if vpc:
logger.info('Creating subnet %s.', self._name)
try:
self._resource = self._ec2.create_subnet(
VpcId=vpc.id,
CidrBlock=self._ctx['cidr_block'],
AvailabilityZone=self._ctx['zone']
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
return
# Wait for the new subnet to become
# available before going any further
waiter = self._client.get_waiter('subnet_available')
waiter.wait(SubnetIds=[self._resource.id])
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_subnet()
else:
logger.error(
'Unable to create subnet %s because VPC %s does not exist.'
' Operation not performed.', self._name, vpc_name
)
def _update_subnet(self):
"""Update a ``Subnet`` object."""
# Set the 'map_public_ip_on_launch' attribute (appears
# as 'Auto-assign Public IP' in the management console)
map_public_ip_on_launch = self._ctx.get('map_public_ip_on_launch', False)
if map_public_ip_on_launch != self._resource.map_public_ip_on_launch:
logger.info(
"Setting 'map_public_ip_on_launch' attribute to '%s' for subnet %s",
map_public_ip_on_launch, self._name
)
try:
self._client.modify_subnet_attribute(
SubnetId=self._resource.id,
MapPublicIpOnLaunch={'Value': map_public_ip_on_launch}
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
self._update_tags()
def assume_role(accountID, rgn, event):
ec2_message = ""
cfn_message = ""
rds_message = ""
client = boto3.client('sts')
response = client.assume_role(RoleArn='arn:aws:iam::'+accountID+':role/'+event['CheckRoleName'],
RoleSessionName='AWSLimits')
session = Session(
aws_access_key_id=response['Credentials']['AccessKeyId'],
aws_secret_access_key=response['Credentials']['SecretAccessKey'],
aws_session_token=response['Credentials']['SessionToken'],
region_name=rgn
)
##############
# call trusted advisor for the limit checks
##############
support_client = session.client('support', region_name='us-east-1')
response = support_client.describe_trusted_advisor_check_result(
checkId='eW7HH0l7J9',
language='en'
)
print "Contacting Trusted Advisor..."
# parse the json and find flagged resources that are in warning mode
flag_list = response['result']['flaggedResources']
warn_list=[]
for fr in flag_list:
if fr['metadata'][5] != "Green":
warn_list.append(fr['metadata'][2]+'\n'+'Region: '+fr['metadata'][0]+'\n------------------------'+'\nResource Limit: '+fr['metadata'][3]+'\n'+'Resource Usage: '+fr['metadata'][4]+'\n')
if not warn_list:
print "TA all green"
else:
global ta_message
ta_message = trustedAlert(warn_list)
###############
#call EC2 limits for rgn
###############
ec2_client = session.client('ec2', region_name=rgn)
response = ec2_client.describe_account_attributes()
attribute_list = response['AccountAttributes']
for att in attribute_list:
if att['AttributeName'] == 'max-instances':
limit_of_instances = att['AttributeValues'][0]['AttributeValue']
print"num of limit: "+limit_of_instances
response = ec2_client.describe_instances(Filters=[{'Name': 'instance-state-name', 'Values':['pending','running']}])
reservation_list = response['Reservations']
num_of_instances = 0
for rsrv in reservation_list:
instance_list = rsrv['Instances']
num_of_instances += len(instance_list)
print "num of instances: "+str(num_of_instances)
#calculate if limit is within threshold
if (float(num_of_instances) / float(limit_of_instances) >= 0.8):
ec2_message = ec2Alert(limit_of_instances, num_of_instances, rgn)
print ec2_message
###############
#cfn resource limit
###############
cfn_client = session.client('cloudformation', region_name=rgn)
# grabbing all stacks except for DELETE_COMPLETE
cfn_response = cfn_client.list_stacks(
StackStatusFilter=[
'CREATE_IN_PROGRESS','CREATE_FAILED','CREATE_COMPLETE',
'ROLLBACK_IN_PROGRESS','ROLLBACK_FAILED','ROLLBACK_COMPLETE',
'DELETE_IN_PROGRESS','DELETE_FAILED','UPDATE_IN_PROGRESS',
'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS','UPDATE_COMPLETE',
'UPDATE_ROLLBACK_IN_PROGRESS','UPDATE_ROLLBACK_FAILED',
'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
'UPDATE_ROLLBACK_COMPLETE'
]
)
done = False
aggregated_results=[]
while not done:
aggregated_results=aggregated_results+cfn_response['StackSummaries']
next_token = cfn_response.get("NextToken", None)
if next_token is None:
done = True
else:
cfn_response = cfn_client.list_stacks(
StackStatusFilter=[
'CREATE_IN_PROGRESS','CREATE_FAILED','CREATE_COMPLETE',
'ROLLBACK_IN_PROGRESS','ROLLBACK_FAILED','ROLLBACK_COMPLETE',
'DELETE_IN_PROGRESS','DELETE_FAILED','UPDATE_IN_PROGRESS',
'UPDATE_COMPLETE_CLEANUP_IN_PROGRESS','UPDATE_COMPLETE',
'UPDATE_ROLLBACK_IN_PROGRESS','UPDATE_ROLLBACK_FAILED',
'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS',
'UPDATE_ROLLBACK_COMPLETE'
],
NextToken=next_token
)
num_of_stacks = len(aggregated_results)
stack_limit = cfn_client.describe_account_limits()
if stack_limit['AccountLimits'][0]['Name'] == 'StackLimit':
limit_of_stacks = stack_limit['AccountLimits'][0]['Value']
if (float(num_of_stacks) / float(limit_of_stacks)) >= 0.8:
cfn_message = cloudformationAlert(limit_of_stacks, num_of_stacks, rgn)
print cfn_message
################
#call RDS Limits for rgn
################
rds_client = session.client('rds', region_name=rgn)
instance_limit = rds_client.describe_account_attributes()
service_limit = instance_limit['AccountQuotas'][0]['Max']
service_usage = instance_limit['AccountQuotas'][0]['Used']
if (float(service_usage) / float(service_limit) >= 0.8):
rds_message = rdsAlert(service_limit, service_usage, rgn)
print rds_message
print "Assumed session for "+accountID+" in region "+rgn
rgn_message = ec2_message + cfn_message + rds_message
return rgn_message;
def __init__(self, access_key, secret_key, region='us-east-1'):
session = Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region)
self.client = session.client('ecs')
self.cloudwatch_event = session.client('events')
self.aws_lambda = session.client('lambda')
def __init__(self, instance_id: str):
session = Session()
self.__instance = session.resource('ec2').Instance(instance_id)
self.__ssm = session.client('ssm')
class DhcpOptionsHandler(base.BaseHandler):
"""Manage ``EC2.DhcpOptions`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra features
# to the base class EC2.DhcpOptions
event = 'creating-resource-class.ec2.DhcpOptions'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a DHCP options set, or update it if it already exists."""
self._resource = self._load(
self._name, manager_name='dhcp_options_sets'
)
if self._resource:
logger.info('DHCP options set %s already exists.', self._name)
else:
self._create_dhcp_options()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a ``DhcpOptions`` object."""
self._resource = self._load(
self._name, manager_name='dhcp_options_sets'
)
if self._resource:
self._update_dhcp_options()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('DHCP options set %s does not exist.', self._name)
def delete_resource(self):
"""Delete a ``DhcpOptions`` object."""
self._resource = self._load(
self._name, manager_name='dhcp_options_sets'
)
if self._resource:
logger.info('Removing DHCP options set %s.', self._name)
try:
self._client.delete_dhcp_options(
DhcpOptionsId=self._resource.id
)
except ClientError as exc:
# If the DHCP options set is associated with a
# VPC, we just log the error and ignore it, as
# the options set object does not provide any
# method to detach itself from the VPC
logger.error(exc)
else:
logger.error('DHCP options set %s does not exist.', self._name)
def _create_dhcp_options(self):
"""Create a ``DhcpOptions`` object."""
dhcp_configurations = []
attrs = ['domain_name_servers',
'domain_name',
'ntp_servers',
'netbios_name_servers',
'netbios_node_type']
for attr in attrs:
if attr in self._ctx:
dhcp_configurations.append({
'Key': attr.replace('_', '-'),
'Values': self._ctx.get(attr, [])
})
logger.info('Creating DHCP options set %s.', self._name)
try:
self._resource = self._ec2.create_dhcp_options(
DhcpConfigurations=dhcp_configurations
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
return
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_dhcp_options()
def _update_dhcp_options(self):
"""Update a ``DhcpOptions`` object."""
self._update_tags()
class RouteTableHandler(base.BaseHandler):
"""Manage ``EC2.RouteTable`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra
# features to the base class EC2.RouteTable
event = 'creating-resource-class.ec2.RouteTable'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a route table, or update it if it already exists."""
self._resource = self._load(self._name, manager_name='route_tables')
if self._resource:
logger.info('Route table %s already exists.', self._name)
else:
self._create_route_table()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a route table object."""
self._resource = self._load(self._name, manager_name='route_tables')
if self._resource:
self._update_route_table()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('Route table %s does not exist.', self._name)
def delete_resource(self):
"""Delete a ``RouteTable`` object."""
self._resource = self._load(self._name, manager_name='route_tables')
if self._resource:
# Remove subnet associations before deleting the route table
for subnet_name in self._ctx.get('subnets') or []:
subnet = self._load(subnet_name, manager_name='subnets')
if subnet:
filters = [
{
'Name': 'association.subnet-id',
'Values': [subnet.id]
},
{
'Name': 'association.route-table-id',
'Values': [self._resource.id]
}
]
associations = list(self._resource.associations.filter(Filters=filters))
if associations:
# The subnet is associated with this route table
try:
associations[0].delete()
except ClientError as exc:
logger.error(exc)
# Remove the route table
logger.info('Removing route table %s.', self._name)
try:
self._client.delete_route_table(RouteTableId=self._resource.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error('Route table %s does not exist.', self._name)
def _create_route_table(self):
"""Create a ``RouteTable`` object."""
vpc_name = self._ctx['vpc']
vpc = self._load(vpc_name, manager_name='vpcs')
if vpc:
logger.info('Creating route table %s.', self._name)
try:
self._resource = self._ec2.create_route_table(VpcId=vpc.id)
except ClientError as exc:
logger.error(exc)
return
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_route_table()
else:
logger.error(
'Unable to create route table %s because VPC %s does not'
' exist. Operation not performed.', self._name, vpc_name
)
def _update_route_table(self):
"""Update a ``RouteTable`` object."""
self._update_tags()
self._update_subnets()
self._update_routes()
def _update_subnets(self):
"""Manage subnet associations for the current route table."""
for subnet_name in self._ctx.get('subnets') or []:
subnet = self._load(subnet_name, manager_name='subnets')
if subnet:
# Search if this subnet is already associated with a route table
filters = [
{
'Name': 'association.subnet-id',
'Values': [subnet.id]
}
]
route_tables = list(self._ec2.route_tables.filter(Filters=filters))
if route_tables:
if route_tables[0].id == self._resource.id:
# The subnet is already associated with this route table
continue
else:
# The subnet is associated with another route table
# and we need to update the association
associations = list(route_tables[0].associations.filter(Filters=filters))
try:
associations[0].replace_subnet(RouteTableId=self._resource.id)
logger.info(
'Subnet %s is now associated with route table %s',
subnet_name, self._name
)
except ClientError as exc:
logger.error(
'An error occured while trying to update route table'
' for subnet %s: %s', subnet_name, exc
)
else:
logger.info(
'Associating subnet %s with route table %s.', subnet_name, self._name
)
try:
self._resource.associate_with_subnet(SubnetId=subnet.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error(
'Unable to associate subnet %s with route table %s because it does'
' not exist. Operation not performed.', subnet_name, self._name
)
def _update_routes(self):
"""Manage routes for the current route table."""
# Routes that are currently set
current_routes = self._resource.custom_routes
# Routes defined in configuration
routes = []
# Routes that do not exist yes
new = []
# If a route is already defined for the given CIDR but has
# to be updated, we need to use a specific client method
modified = []
# Routes that are no longer defined in the configuration
trash = []
# Parse the configuration and populate route list
for item in self._ctx.get('routes') or []:
route = self._build_route(item)
if route:
routes.append(RouteWrapper(route))
# Search for new routes and routes that should be updated
for item in set(routes) - set(current_routes):
match = next(
(r for r in current_routes if r.destination == item.destination), None
)
if match:
modified.append(item)
else:
new.append(item)
# Search for routes that should be deleted
for item in set(current_routes) - set(routes):
match = next(
(r for r in routes if r.destination == item.destination), None
)
if not match:
trash.append(item)
for wrapper in modified:
logger.info('%s: replacing route - %s.', self._name, wrapper)
if all((wrapper.instance_id, wrapper.network_interface_id)):
# We must only pass the ``instance_id`` attribute in
# order to update a route with an EC2 instance target
del wrapper.network_interface_id
try:
self._client.replace_route(RouteTableId=self._resource.id, **wrapper.route)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
for wrapper in new:
logger.info('%s: creating route - %s.', self._name, wrapper)
if all((wrapper.instance_id, wrapper.network_interface_id)):
# We must only pass the ``instance_id`` attribute in
# order to create a route with an EC2 instance target
del wrapper.network_interface_id
try:
self._resource.create_route(**wrapper.route)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
for wrapper in trash:
logger.info('%s: removing route - %s.', self._name, wrapper)
try:
self._client.delete_route(
RouteTableId=self._resource.id,
DestinationCidrBlock=wrapper.destination
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
def _build_route(self, item):
"""Load target objects for route definitions and return a route as a dictionary.
The following targets are supported:
- internet gateways
- virtual private gateways
- VPC peering connections
- EC2 instances
"""
filters = [{'Name': 'tag:Name', 'Values': [item['target_name']]}]
# internet gateway targets
if item['target_type'] == 'internet_gateway':
resp = self._client.describe_internet_gateways(Filters=filters)
gateways = resp.get('InternetGateways')
igw = gateways[0] if gateways else None
if igw:
return {
'GatewayId': igw['InternetGatewayId'],
'DestinationCidrBlock': item['destination']
}
# virtual private gateway targets
elif item['target_type'] == 'vpn_gateway':
resp = self._client.describe_vpn_gateways(Filters=filters)
gateways = resp.get('VpnGateways')
vgw = gateways[0] if gateways else None
if vgw:
return {
'GatewayId': vgw['VpnGatewayId'],
'DestinationCidrBlock': item['destination']
}
# VPC peering connection targets
elif item['target_type'] == 'vpc_peering_connection':
resp = self._client.describe_vpc_peering_connections(Filters=filters)
connections = resp.get('VpcPeeringConnections')
connection = connections[0] if connections else None
if connection:
return {
'VpcPeeringConnectionId': connection['VpcPeeringConnectionId'],
'DestinationCidrBlock': item['destination']
}
# EC2 instance targets
elif item['target_type'] == 'instance':
resp = self._client.describe_instances(Filters=filters)
reservations = resp.get('Reservations')
reservation = reservations[0] if reservations else {}
instances = reservation.get('Instances')
instance = instances[0] if instances else None
if instance:
route = {
'InstanceId': instance['InstanceId'],
'DestinationCidrBlock': item['destination']
}
interfaces = instance.get('NetworkInterfaces')
interface = interfaces[0] if interfaces else None
if interface:
route['NetworkInterfaceId'] = interface['NetworkInterfaceId']
return route
# Unsupported targets
else:
logger.error("%s: unsupported target type '%s'", self._name, item['target_type'])
return None
class InternetGatewayHandler(base.BaseHandler):
"""Manage ``EC2.InternetGateway`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra features
# to the base class EC2.InternetGateway
event = "creating-resource-class.ec2.InternetGateway"
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource("ec2")
self._client = self._session.client("ec2")
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get("name")
def create_resource(self):
"""Create an internet gateway, or update it if it already exists."""
self._resource = self._load(self._name, manager_name="internet_gateways")
if self._resource:
logger.info("Internet gateway %s already exists.", self._name)
else:
self._create_internet_gateway()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update an internet gateway object."""
self._resource = self._load(self._name, manager_name="internet_gateways")
if self._resource:
self._update_internet_gateway()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error("Internet gateway %s does not exist.", self._name)
def delete_resource(self):
"""Delete an ``InternetGateway`` object."""
self._resource = self._load(self._name, manager_name="internet_gateways")
if self._resource:
logger.info("Removing internet gateway %s.", self._name)
if self._resource.attachments:
# If the internet gateway is attached to a VPC,
# we don't take the responsibility to detach it
vpc_id = self._resource.attachments[0]["VpcId"]
logger.warning(
"The internet gateway %s is currently attached to VPC %s." " Operation not performed.",
self._name,
vpc_id,
)
else:
try:
self._client.delete_internet_gateway(InternetGatewayId=self._resource.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error("Internet gateway %s does not exist.", self._name)
def _create_internet_gateway(self):
"""Create an ``InternetGateway`` object."""
logger.info("Creating internet gateway %s.", self._name)
try:
self._resource = self._ec2.create_internet_gateway()
except ClientError as exc:
logger.error(exc)
return
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_internet_gateway()
def _update_internet_gateway(self):
"""Update a ``InternetGateway`` object."""
self._update_tags()
class AWSClient(object):
"""Manages automatically creating and destroying clients to AWS services."""
def __init__(self, resource, config, credentials=None, region_name=None):
"""Constructor
:param resource: AWS specific token for resource type. e.g., 's3', 'sqs', etc.
:type resource: string
:param config: Resource specific configuration
:type config: :class:`botocore.client.Config`
:param credentials: Authentication values needed to access AWS. If no credentials are passed, then IAM
role-based access is assumed.
:type credentials: :class:`util.aws.AWSCredentials`
:param region_name: The AWS region the resource resides in.
:type region_name: string
"""
self.credentials = credentials
self.region_name = region_name
self._client = None
self._resource_name = resource
self._config = config
def __enter__(self):
"""Callback handles creating a new client for AWS access."""
logger.debug('Setting up AWS client...')
session_args = {}
if self.credentials:
session_args['aws_access_key_id'] = self.credentials.access_key_id
session_args['aws_secret_access_key'] = self.credentials.secret_access_key
if self.region_name:
session_args['region_name'] = self.region_name
self._session = Session(**session_args)
self._client = self._session.client(self._resource_name, config=self._config)
self._resource = self._session.resource(self._resource_name, config=self._config)
return self
def __exit__(self, type, value, traceback):
"""Callback handles destroying an existing client."""
pass
@staticmethod
def instantiate_credentials_from_config(config):
"""Extract credential keys from configuration and return instantiated credential object
If values provided in the `access_key_id` or `secret_access_key` keys of `credentials` configuration object are
empty, None will be returned. In this case, role-based authentication should be attempted.
:param config: Resource specific configuration
:type config: :class:`botocore.client.Config`
:return: instantiated credential object or None if keys provided were empty
:rtype: :class:`util.aws.AWSCredentials`
:raises :class:`util.exceptions.InvalidAWSCredentials`: If the credentials provided are incomplete.
"""
if 'credentials' in config and config['credentials']:
credentials_dict = config['credentials']
if 'access_key_id' not in credentials_dict:
raise InvalidAWSCredentials('"credentials" requires "access_key_id" to be populated')
if 'secret_access_key' not in credentials_dict:
raise InvalidAWSCredentials('"credentials" requires "secret_access_key" to be populated')
access_key = credentials_dict['access_key_id'].strip()
secret_key = credentials_dict['secret_access_key'].strip()
# If either Access Key or Secret Access Key are empty, fail-over to role-based auth.
# TODO: This should be changed to also raise as above, once the UI has been improved to prune unset values.
if not len(access_key) or not len(secret_key):
return None
return AWSCredentials(access_key, secret_key)
