def __init__(self):
self.logger = get_logger(__name__)
region_name = os.getenv('AWS_REGION', 'ap-northeast-1')
# 環境変数 AWS_REGION が設定されていない場合は処理しない
if region_name != '':
session = Session(region_name=region_name)
self.resource = session.resource('s3')
def gcp_resource():
sesh = Session(aws_access_key_id=conf.GCP_ACCESS_KEY,
aws_secret_access_key=conf.GCP_SECRET_KEY)
sesh.events.unregister(
'before-parameter-build.s3.ListObjects',
set_list_objects_encoding_type_url)
return sesh.resource('s3', endpoint_url=conf.GCP_ENDPOINT)
def test_create_subnet(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
subnet_ctx = {
'name': 'subnet01a',
'cidr_block': '10.0.10.0/25',
'zone': 'us-west-2a',
'vpc': 'vpc01',
'tags': {
'description': 'Test subnet (zone a) for VPC vpc01'
}
}
tags = [
{
'Key': 'Name',
'Value': 'subnet01a'
},
{
'Key': 'Description',
'Value': 'Test subnet (zone a) for VPC vpc01'
}
]
vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01a']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_subnet.SubnetWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Subnet'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=vpc_filters))
vpc = vpcs[0]
# Create the subnet
h = ec2_subnet.create_handler(subnet_ctx, self.credentials)
h.create_resource()
subnets = list(ec2.subnets.filter(Filters=subnet_filters))
subnet = subnets[0]
self.assertEqual(len(subnets), 1)
self.assertEqual(subnet.name, 'subnet01a')
self.assertEqual(subnet.cidr_block, '10.0.10.0/25')
self.assertEqual(subnet.availability_zone, 'us-west-2a')
self.assertEqual(subnet.vpc_id, vpc.id)
self.assertEqual(subnet.map_public_ip_on_launch, False)
self.assertCountEqual(subnet.tags, tags)
def test_delete_internet_gateway(self):
ctx = {
'name': 'igw01'
}
filters = [{'Name': 'tag:Name', 'Values': ['igw01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_igw.InternetGatewayWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.InternetGateway'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the internet gateway
h = ec2_igw.create_handler(ctx, self.credentials)
h.create_resource()
gateways = list(ec2.internet_gateways.filter(Filters=filters))
self.assertEqual(len(gateways), 1)
# We clear the resource cache to simulate a new
# program execution with the 'delete' option
base.BaseHandler._cache.clear()
# Delete the internet gateway
h.delete_resource()
gateways = list(ec2.internet_gateways.filter(Filters=filters))
self.assertEqual(len(gateways), 0)
def to_parquet(
df,
bucket_name,
prefix,
retry_config,
session_kwargs,
client_kwargs,
compression=None,
flavor="spark",
):
import pyarrow as pa
from pyarrow import parquet as pq
session = Session(**session_kwargs)
client = session.resource("s3", **client_kwargs)
bucket = client.Bucket(bucket_name)
table = pa.Table.from_pandas(df)
buf = pa.BufferOutputStream()
pq.write_table(table, buf, compression=compression, flavor=flavor)
response = retry_api_call(
bucket.put_object,
config=retry_config,
Body=buf.getvalue().to_pybytes(),
Key=prefix + str(uuid.uuid4()),
)
return "s3://{0}/{1}".format(response.bucket_name, response.key)
def uploadFile(url):
filename = url.split("/")[-1]
fileextension = filename.split('.')[1]
file = requests.get(url).content
filepath = os.path.join(MEDIA_ROOT, filename)
with open(filepath, 'wb') as destination:
destination.write(file)
file = open(filepath, 'rb')
extension = FileUploadUtils.getContentType(fileextension)
valid_file = True
if extension is None:
valid_file = False
session = Session(aws_access_key_id=settings.AWS_ACCESS_KEY,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=settings.AWS_REGION_NAME)
s3 = session.resource('s3')
file_key = FileUploadUtils.getFileKey()
if valid_file:
res = s3.Bucket(settings.AWS_BUCKET_NAME).put_object(Key=file_key, Body=file, ContentType=extension,
ACL='public-read')
data = {'key': file_key, 'file_name': filename, 'is_link': True}
serializer = serializers.CreateFileUploadSerializer(data=data)
if serializer.is_valid():
serializer.save()
if os.path.isfile(filepath):
os.remove(filepath)
media = Media.objects.get(key=file_key)
return media
else:
return None
def lambda_handler(event, context):
# Resulted list
ips = []
# Init Boto3 session
sess = Session()
asg_client = sess.client("autoscaling")
ec2res = sess.resource("ec2")
# Give me all the AutoScaling groups!
for asg in asg_client.describe_auto_scaling_groups()['AutoScalingGroups']:
# Give me all the instances in the AutoScaling group!
for instance in asg['Instances']:
# Is you healthy?
if instance['HealthStatus'] == "Healthy":
# Give me ze private address
instance = ec2res.Instance(instance['InstanceId'])
ips.append(instance.private_ip_address)
return {
"statusCode": 200,
"body": json.dumps({
"ips": ips,
}),
}
def test_delete_vpc(self):
ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_vpc.VpcWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Vpc'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=filters))
self.assertEqual(len(vpcs), 1)
# We clear the resource cache to simulate a new
# program execution with the 'delete' option
base.BaseHandler._cache.clear()
# Delete the VPC
h.delete_resource()
vpcs = list(ec2.vpcs.filter(Filters=filters))
self.assertEqual(len(vpcs), 0)
def generate_dummy_bucket():
session = Session(aws_access_key_id=DUMMY_ACCESS_KEY_ID,
aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY)
s3 = session.resource('s3', region_name='us-east-1')
# create a bucket
s3.create_bucket(Bucket=DUMMY_BUCKET_NAME)
# create some objects in this bucket
# the root contain 7 objects: 4 files and 3 directories
# the first entry in the tree should be the directory `cache`, since boto3
# returns the values sorted alphabetically. The last object should be the
# file called `index.js`.
files = ('index.js', 'avatar.jpg', '__init__.py', 'Makefile',
'css/app.less', 'css/base.css', 'js/vendor/angular.js',
'js/vendor/angular.min.js', 'js/vendor/latest/react.js',
'cache/foo.rb', 'cache/staticfiles/latest/index.js'
'cache/staticfiles/dummy.txt', 'cache/staticfiles/logs.txt')
for file in files:
content = string.ascii_letters * random.randint(1, 10)
s3.meta.client.put_object(Bucket=DUMMY_BUCKET_NAME,
Key=file,
Body=content)
def to_parquet(
df: "DataFrame",
bucket_name: str,
prefix: str,
retry_config: RetryConfig,
session_kwargs: Dict[str, Any],
client_kwargs: Dict[str, Any],
compression: str = None,
flavor: str = "spark",
) -> str:
import pyarrow as pa
from pyarrow import parquet as pq
session = Session(**session_kwargs)
client = session.resource("s3", **client_kwargs)
bucket = client.Bucket(bucket_name)
table = pa.Table.from_pandas(df)
buf = pa.BufferOutputStream()
pq.write_table(table, buf, compression=compression, flavor=flavor)
response = retry_api_call(
bucket.put_object,
config=retry_config,
Body=buf.getvalue().to_pybytes(),
Key=prefix + str(uuid.uuid4()),
)
return f"s3://{response.bucket_name}/{response.key}"
def main(sg_name="asgard.ai", action="add", ip=None):
""" """
# parse args
assert action in ["add", "remove"]
# get aws credential from env
aws_region = os.environ.get("AWS_REGION", "eu-west-1")
aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"]
aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"]
# aws client
session = Session(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=aws_region
)
ec2 = session.resource('ec2')
# my ip & security group
my_ip = get_my_ip() + '/32'
sg = get_security_group(ec2, name=sg_name)
already_authorized_ip = list(
set([x["CidrIp"] for y in sg.ip_permissions for x in y["IpRanges"]])
)
# add or remove my ip form security group
if action == "add" and my_ip not in already_authorized_ip:
add_ip_to_security_group(sg, my_ip)
if ip is not None:
add_ip_to_security_group(sg, ip + "/32")
elif action == "remove" and my_ip in already_authorized_ip:
remove_ip_to_security_group(sg, my_ip)
if ip is not None:
remove_ip_to_security_group(sg, ip + "/32")
def __init__(self,
bucket_name,
path=None,
aws_access_key_id=None,
aws_secret_access_key=None):
# try to get the access key and secret key either from this object's
# init, or from the global config.
self._access_key = aws_access_key_id or config.aws_access_key_id
self._secret_key = aws_secret_access_key or config.aws_secret_access_key
if not (self._access_key and self._secret_key):
raise ImproperlyConfiguredError
# create a session using the credentials
session = Session(aws_access_key_id=self._access_key,
aws_secret_access_key=self._secret_key)
# create an S3 resource
self.s3 = session.resource(self.BOTO3_S3_RESOURCE_ID)
self.client = self.s3.meta.client
# ensure that the bucket exists, and then set the bucket name
self.__ensure_bucket_exists(bucket_name)
self.bucket_name = bucket_name
# set the base path
self.path = normalize_path(path)
# set containers
self.directories = []
self.files = []
# get the base tree and prepare the iterable
self.__tree = self.__fetch_tree()
def copy_dynamo_schema(src_session: Session, dst_session: Session, tables: list = ()) -> None:
"""
Reads the DynamoDB in source region and creates the same tables in destination region.
:param src_session: The aws session object of the source profile/region.
:param dst_session: The aws session object of the destination profile/region.
:param tables: A list of the tables to be copied. If empty all tables will be created.
:return: None
"""
d_src_client = src_session.client('dynamodb')
if not tables:
tables = d_src_client.list_tables()['TableNames']
for name in tables:
d_src_resource = src_session.resource('dynamodb')
table = d_src_resource.Table(name)
d_dest_resource = dst_session.resource('dynamodb')
provisioned_throughput = table.provisioned_throughput
if 'NumberOfDecreasesToday' in provisioned_throughput.keys():
del provisioned_throughput['NumberOfDecreasesToday']
params = {
'TableName': name,
'KeySchema': table.key_schema,
'AttributeDefinitions': table.attribute_definitions,
'ProvisionedThroughput': provisioned_throughput
}
if table.global_secondary_indexes:
gsi = table.global_secondary_indexes
gsi_copy = copy.deepcopy(gsi)
for item in gsi:
for key in item.keys():
if key not in ["IndexName", "KeySchema", "Projection", "ProvisionedThroughput"]:
del gsi_copy[gsi.index(item)][key]
if 'NumberOfDecreasesToday' in gsi_copy[gsi.index(item)]['ProvisionedThroughput'].keys():
del gsi_copy[gsi.index(item)]['ProvisionedThroughput']['NumberOfDecreasesToday']
params['GlobalSecondaryIndexes'] = gsi_copy
d_dest_resource.create_table(**params)
print(f'Created table {name}')
def __init__(self, config=None):
Indexer.__init__(self, config=config)
session = Session(
aws_access_key_id=config['aws_access_key_id'],
aws_secret_access_key=config['aws_secret_access_key'],
region_name=config['aws_region'])
ddb = session.resource('dynamodb')
self.table = ddb.Table('nico_crawler')
def __init__(self, env_session: boto3.Session):
self._s3 = env_session.client(
's3',
config=botocore.client.Config(
max_pool_connections=S3Service._MAX_POOL_SIZE))
self._s3_resource = env_session.resource('s3')
self.__progress_total = 0
self.lock = Lock()
def __init__(self):
self.logger = get_logger(__name__)
region_name = os.getenv('AWS_REGION', 'ap-northeast-1')
# 環境変数 AWS_REGION が設定されていない場合は処理しない
if region_name != '':
session = Session(region_name=region_name)
self.resource = session.resource('dynamodb')
self.default_table = 'bookbot-entry'
def test_create_dhcp_options(self):
ctx = {
'name': 'dhcp01',
'domain_name': [
'test01.us-west-2.aws'
],
'domain_name_servers': [
'10.0.10.2'
],
'tags': {
'description': 'DHCP options set for VPC vpc01'
}
}
tags = [
{
'Key': 'Name',
'Value': 'dhcp01'
},
{
'Key': 'Description',
'Value': 'DHCP options set for VPC vpc01'
}
]
dhcp_configurations = [
{
'Key': 'domain-name',
'Values': [{'Value': 'test01.us-west-2.aws'}]
},
{
'Key': 'domain-name-servers',
'Values': [{'Value': '10.0.10.2'}]
}
]
filters = [{'Name': 'tag:Name', 'Values': ['dhcp01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_dhcp.DhcpOptionsWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.DhcpOptions'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the DHCP options set
h = ec2_dhcp.create_handler(ctx, self.credentials)
h.create_resource()
dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters))
dhcp = dhcp_options_sets[0]
self.assertEqual(len(dhcp_options_sets), 1)
self.assertEqual(dhcp.name, 'dhcp01')
self.assertCountEqual(dhcp.dhcp_configurations, dhcp_configurations)
self.assertCountEqual(dhcp.tags, tags)
def read_dynamodb_data(table, recette, profile):
"""
Scan all item from dynamodb.
:param table: String
:return: Data in Dictionary Format.
"""
print('Connecting to AWS DynamoDb')
session = Session(profile_name=profile)
dynamodb_resource = session.resource('dynamodb')
table = dynamodb_resource.Table(table)
print('Downloading ', end='')
keys = []
for item in table.attribute_definitions:
keys.append(item['AttributeName'])
keys_set = set(keys)
item_count = table.item_count
if recette is not None:
raw_data = table.scan(FilterExpression=Attr("codeRecette").eq(recette))
else:
raw_data = table.scan(FilterExpression=Attr("noEtape").eq("0"))
if raw_data is None:
return None
items = raw_data['Items']
print('This is a ' + str(type(items)))
items = strip_ssml_tags(items)
fieldnames = set([]).union(get_keys(items))
cur_total = (len(items) + raw_data['Count'])
if cur_total > item_count:
percents = 99.99
else:
percents = cur_total * 100 / item_count
print("{} records ..... {:02.0f}%".format(raw_data['Count'], percents),
end='\r')
while raw_data.get('LastEvaluatedKey'):
print('Downloading ', end='')
raw_data = table.scan(ExclusiveStartKey=raw_data['LastEvaluatedKey'])
items.extend(raw_data['Items'])
fieldnames = fieldnames.union(get_keys(items))
cur_total = (len(items) + raw_data['Count'])
if cur_total > item_count:
percents = 99.99
else:
percents = cur_total * 100 / item_count
print("{} records ..... {:02.0f}%".format(raw_data['Count'], percents),
end='\r')
print()
print("Total downloaded records: {}".format(len(items)))
for fieldname in fieldnames:
if fieldname not in keys_set:
keys.append(fieldname)
return {'items': items, 'keys': keys}
def test_delete_route_table_with_association(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
subnet_ctx = {
'name': 'subnet01a',
'cidr_block': '10.0.10.0/25',
'zone': 'us-west-2a',
'vpc': 'vpc01'
}
rt_ctx = {
'name': 'rt01',
'vpc': 'vpc01',
'subnets': [
'subnet01a'
]
}
filters = [{'Name': 'tag:Name', 'Values': ['rt01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_rt.RouteTableWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.RouteTable'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the subnet
h = ec2_subnet.create_handler(subnet_ctx, self.credentials)
h.create_resource()
# Create the route table
h = ec2_rt.create_handler(rt_ctx, self.credentials)
h.create_resource()
route_tables = list(ec2.route_tables.filter(Filters=filters))
self.assertEqual(len(route_tables), 1)
# We clear the resource cache to simulate a new
# program execution with the 'delete' option
base.BaseHandler._cache.clear()
# Delete the route table
h.delete_resource()
route_tables = list(ec2.route_tables.filter(Filters=filters))
self.assertEqual(len(route_tables), 0)
def __ec2_detach_eks(session: Session, kubecontext: str, namespace: str,
label_name: str) -> str:
"""
Utility method to detach instance from a given namespace. This returns a node IP, gets instance ID from the
same and returns back the same for chaos operations
:param session:
:param kubecontext:
:param namespace:
:param label_name:
:return:
"""
logger.info("Getting list of pods for namespace " + namespace)
v1 = K8sUtils.init_k8s_client(kubecontext)
ip_address_list = []
v1podlist = v1.list_namespaced_pod(namespace,
label_selector=label_name).items
if not v1podlist:
raise ChaosTestException(
"No pods are there in the namespace which was provided " +
namespace)
if len(v1podlist) == 0:
raise ChaosTestException("List of pods is empty for namespace " +
namespace)
for v1pod in v1podlist:
if v1pod.status.phase == "Running":
ip_address_list.append(v1pod.status.host_ip)
if len(ip_address_list) == 0:
raise ChaosTestException(
"nodes are not present matching pod label " + label_name)
ip_address_list = list(dict.fromkeys(ip_address_list))
# list_node_ip_addresses = []
# http_nodes = v1.list_node(pretty='true')
#
# for i in range(len(http_nodes.items) - 1):
# address_list = http_nodes.items[i].status.addresses
# for address in address_list:
# if address.type == 'InternalIP':
# list_node_ip_addresses.append(address.address)
ip_selected = random.choice(ip_address_list)
ec2_resource = session.resource('ec2')
response = ec2_resource.instances.filter(Filters=[{
'Name': 'instance-state-name',
'Values': ['running']
}])
instance_id = ''
for instance in response:
if ip_selected == instance.private_ip_address:
logger.info("Instance Id returned " + instance.id)
instance_id = instance.id
break
if not instance_id:
raise Exception("Not able to get instance id, exiting")
return instance_id
def test_function(event, context):
get_google_response()
aws_session = Session(region_name=REGION)
sqs = aws_session.resource('sqs')
sqs_queue = sqs.get_queue_by_name(QueueName=QUEUE)
print('SQS url:' + sqs_queue.url)
sqs_queue.send_message(MessageBody='hey Jude')
return 'success'
def import_dynamodb_items_to_es(table_name, aws_secret, aws_access, aws_region,
event_source_arn, lambda_f, scan_limit):
global reports
global partSize
global object_amount
logger = logging.getLogger()
logger.setLevel(logging.INFO)
session = Session(aws_access_key_id=aws_access,
aws_secret_access_key=aws_secret,
region_name=aws_region)
dynamodb = session.resource('dynamodb')
logger.info('dynamodb: %s', dynamodb)
ddb_table_name = table_name
table = dynamodb.Table(ddb_table_name)
logger.info('table: %s', table)
ddb_keys_name = [a['AttributeName'] for a in table.attribute_definitions]
logger.info('ddb_keys_name: %s', ddb_keys_name)
response = None
while True:
if not response:
response = table.scan(Limit=scan_limit)
else:
response = table.scan(
ExclusiveStartKey=response['LastEvaluatedKey'],
Limit=scan_limit)
for i in response["Items"]:
ddb_keys = {k: i[k] for k in i if k in ddb_keys_name}
ddb_data = boto3.dynamodb.types.TypeSerializer().serialize(i)["M"]
ddb_keys = boto3.dynamodb.types.TypeSerializer().serialize(
ddb_keys)["M"]
record = {
"dynamodb": {
"SequenceNumber": "0000",
"Keys": ddb_keys,
"NewImage": ddb_data
},
"awsRegion": aws_region,
"eventName": "INSERT",
"eventSourceARN": event_source_arn,
"eventSource": "aws:dynamodb"
}
partSize += 1
object_amount += 1
logger.info(object_amount)
reports.append(record)
if partSize >= 100:
send_to_eslambda(reports, lambda_f)
if 'LastEvaluatedKey' not in response:
break
if partSize > 0:
send_to_eslambda(reports, lambda_f)
def boto_resource_dynamodb(
session: _boto3.Session,
opts: _intf.Options,
) -> _tp.Any:
kw = opts.mapped_update({
"region": "region_name",
"dynamodb_endpoint": "endpoint_url",
})
return session.resource("dynamodb", **kw)
def test_delete_attached_internet_gateway(self):
igw_ctx = {
'name': 'igw01'
}
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'internet_gateway': 'igw01'
}
igw_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}]
vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_igw.InternetGatewayWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.InternetGateway'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the internet gateway
h = ec2_igw.create_handler(igw_ctx, self.credentials)
h.create_resource()
gateways = list(ec2.internet_gateways.filter(Filters=igw_filters))
igw = gateways[0]
self.assertCountEqual(igw.attachments, [])
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=vpc_filters))
vpc = vpcs[0]
# Test that the internet gateway has been attached
igw.reload()
attachments = [{'VpcId': vpc.id, 'State': 'available'}]
self.assertCountEqual(igw.attachments, attachments)
# We clear the resource cache to simulate a new
# program execution with the 'delete' option
base.BaseHandler._cache.clear()
# Delete the internet gateway
h = ec2_igw.create_handler(igw_ctx, self.credentials)
h.delete_resource()
gateways = list(ec2.internet_gateways.filter(Filters=igw_filters))
# The gateway was not deleted
self.assertEqual(len(gateways), 1)
class Resource_Ec2 :
def __init__(self,iamProfilename) :
from boto3 import Session
self.session = Session(profile_name=iamProfilename)
self.ec2 = self.session.resource('ec2')
#Get EC2 instance list
def _getEntireListofCurrentInstances(self) :
response = [{'instance.id':instance.id,'instance.state':instance.state} for instance in self.ec2.instances.all()]
return(response)
#Create and Run EC2 instance
def _createNewInstances(self,imageId,minCount,maxCount,instanceType,keyName,securityGroupIds,iamInstanceProfile,instanceTags,userData) :
instance = self.ec2.create_instances(
#create_instances Arguments References : http://docs.aws.amazon.com/ko_kr/cli/latest/userguide/generate-cli-skeleton.html
ImageId = imageId,
MinCount = minCount,
MaxCount = maxCount,
InstanceType = instanceType,
KeyName = keyName,
SecurityGroupIds = securityGroupIds,
IamInstanceProfile = iamInstanceProfile,
TagSpecifications = [
{
'ResourceType': 'instance',
#ResourceType SHOULD BE in TagSpecifications
'Tags': [
instanceTags,
]
}
],
UserData = userData,
)
return(instance)
#Stop EC2 instances
def _stopInstances(self,instanceIds) :
instance = self.ec2.instances.filter(InstanceIds=instanceIds)
response = instance.stop()
return(response)
#Start EC2 instances
def _startInstances(self,instanceIds) :
instance = self.ec2.instances.filter(InstanceIds=instanceIds)
#Case#01 instance type is not supported on specific AWS region(e.g. ap-northeast-2) : botocore.exceptions.ClientError: An error occurred (Unsupported) when calling the StartInstances operation: The requested configuration is currently not supported. Please check the documentation for supported configurations.
#Case#01 Trouble Shooting : chek the limits and change the instance type available, https://docs.aws.amazon.com/ko_kr/AWSEC2/latest/UserGuide/ec2-resource-limits.html
#Case#01 Ref : m5.large is not supported on ap-northeaset-2, https://aws.amazon.com/ko/blogs/korea/m5-the-next-generation-of-general-purpose-ec2-instances
response = instance.start()
return(response)
#Terminate EC2 instances
def _terminateInstances(self,instanceIds) :
instance = self.ec2.instances.filter(InstanceIds=instanceIds)
response = instance.terminate()
return(response)
def aws_connectors():
session = Session(
region_name=os.environ.get('AWS_DEFAULT_REGION'),
aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'),
aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY'))
sqs = session.resource('sqs')
sqs_client = session.client('sqs')
return sqs, sqs_client
def import_xls(request):
form = Upload_XLForm(request.POST or None, request.FILES or None)
if form.is_valid():
form.save()
form = Upload_XLForm()
obj = Upload_XL.objects.get(activated=False)
s3_session = Session(aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
bucket_object = s3_session.resource('s3').Bucket(
AWS_STORAGE_BUCKET_NAME).Object(str(obj.file_name))
content = bucket_object.get()['Body'].read()
workbook = xlrd.open_workbook_xls(file_contents=content)
# book = xlrd.open_workbook(obj.file_name.url)
sh = workbook.sheet_by_index(0)
p_row = 1
while p_row < sh.nrows:
# print(sh.row(p_row).value)
user, created = User.objects.get_or_create(
username=sh.row(p_row)[3].value,
email=sh.row(p_row)[4].value,
)
current_post = Post.objects.create(
title=sh.row(p_row)[0].value,
content=sh.row(p_row)[1].value,
date_posted=datetime.datetime.strptime(
sh.row(p_row)[2].value, "%Y-%m-%d %H:%M:%S"),
author=user,
likes=sh.row(p_row)[5].value,
)
c_row = p_row + 3
try:
while sh.row(c_row)[1].value != '':
comment_user, c_created = User.objects.get_or_create(
username=sh.row(c_row)[3].value,
email=sh.row(c_row)[4].value,
)
Comment.objects.create(
post=current_post,
author=comment_user,
text=sh.row(c_row)[1].value,
created_date=datetime.datetime.strptime(
sh.row(c_row)[2].value, "%Y-%m-%d %H:%M:%S"),
)
c_row += 1
except:
pass
p_row = c_row + 2
obj.activated = True
obj.save()
obj.delete()
return render(request, 'blog/uploadXL.html', {'form': form})
def test_create_route_table(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
rt_ctx = {
'name': 'rt01',
'vpc': 'vpc01',
'tags': {
'description': 'Replace the default route table for VPC vpc01'
}
}
tags = [
{
'Key': 'Name',
'Value': 'rt01'
},
{
'Key': 'Description',
'Value': 'Replace the default route table for VPC vpc01'
}
]
vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
rt_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_rt.RouteTableWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.RouteTable'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=vpc_filters))
vpc = vpcs[0]
# Create the route table
h = ec2_rt.create_handler(rt_ctx, self.credentials)
h.create_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt_filters))
rt = route_tables[0]
self.assertEqual(len(route_tables), 1)
self.assertEqual(rt.name, 'rt01')
self.assertEqual(rt.vpc_id, vpc.id)
self.assertCountEqual(rt.tags, tags)
def test_create_alert(self):
session = Session()
# Get the service resource
dynamodb = session.resource('dynamodb')
dynamodb.create_table(
TableName='sso-dashboard-alert1',
KeySchema=[
{
'AttributeName': 'alert_id',
'KeyType': 'HASH'
}
],
AttributeDefinitions=[
{
'AttributeName': 'alert_id',
'AttributeType': 'S'
}
],
ProvisionedThroughput={
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
}
)
self.table = dynamodb.Table('sso-dashboard-alert1')
self.table.meta.client.get_waiter('table_exists').wait(TableName='sso-dashboard-alert1')
a = alert.Alert()
a.dynamodb = self.table
alert_dict = {
'user_id': 'bob|ad|123456',
'message': 'foo',
'severity': 'HIGH'
}
#
res = a.create(alert_dict=alert_dict)
sample_alert_id = res['Attributes']['alert_id']
assert res['ResponseMetadata']['HTTPStatusCode'] == 200
res = a.find('bob|ad|123456')
assert len(res) is 1
updated_alert_dict = {
'user_id': 'bob|ad|123456',
'message': 'foo',
'severity': 'MEDIUM'
}
res = a.update(alert_id=sample_alert_id, alert_dict=updated_alert_dict)
assert res['ResponseMetadata']['HTTPStatusCode'] == 200
res = a.destroy(alert_id=sample_alert_id, user_id='bob|ad|123456')
assert res['ResponseMetadata']['HTTPStatusCode'] == 200
def ReadBucket(bucket_type, bucket_name, prefix=''):
from boto3 import Session
from google.cloud import storage
try:
blobs = []
# AWS S3
if bucket_type == 's3' or bucket_type == 'aws':
#download_url = f"http://{bucket_name}.s3.amazonaws.com/"
s3_client = Session()
s3_resource = s3_client.resource('s3')
s3_bucket = s3_resource.Bucket(bucket_name)
for obj in s3_bucket.objects.filter(Prefix=prefix):
#print(obj.__dict__)
blobs.append(obj.key)
# Google Cloud Storage
if bucket_type == 'gs' or bucket_type == 'gcs':
#download_url = f"http://{bucket_name}.storage.googleapis.com/"
storage_client = storage.Client.create_anonymous_client()
for blob in storage_client.list_blobs(bucket_name, prefix=prefix):
#print(blob.__dict__)
blobs.append(blob.name)
files = []
dirs = {}
for blob in blobs:
#print(blob)
file_name = blob.split('/')[-1]
if '/' in blob:
parent_dir = blob.split('/')[-2]
else:
parent_dir = ""
path_match = prefix.split('/')[-1]
print(file_name, parent_dir, path_match)
if parent_dir == path_match:
files.append(file_name)
if '/' in blob:
# Directory handling
_ = blob.replace(prefix, '')
if '/' in _:
_ = _.split('/')[-2]
print(_)
if not _ in dirs:
dirs[_] = True
return dict(files=files, dirs=dirs)
except Exception as e:
raise (e)
def setup_resources(self):
if self.aws_access_key_id and self.aws_secret_access_key:
session = Session(aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.aws_region_name)
self.resource = session.resource('dynamodb')
else:
self.resource = boto3.resource('dynamodb',
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.aws_region_name)
self.table = self.resource.Table(self.table_name)
class AWSClient(object):
"""Manages automatically creating and destroying clients to AWS services."""
def __init__(self, resource, config, credentials=None, region_name=None):
"""Constructor
:param resource: AWS specific token for resource type. e.g., 's3', 'sqs', etc.
:type resource: string
:param config: Resource specific configuration
:type config: :class:`botocore.client.Config`
:param credentials: Authentication values needed to access AWS. If no credentials are passed, then IAM
role-based access is assumed.
:type credentials: :class:`util.aws.AWSCredentials`
:param region_name: The AWS region the resource resides in.
:type region_name: string
"""
self.credentials = credentials
self.region_name = region_name
self._client = None
self._resource_name = resource
self._config = config
def __enter__(self):
"""Callback handles creating a new client for AWS access."""
logger.debug('Setting up AWS client...')
session_args = {}
if self.credentials:
session_args['aws_access_key_id'] = self.credentials.access_key_id
session_args['aws_secret_access_key'] = self.credentials.secret_access_key
if self.region_name:
session_args['region_name'] = self.region_name
self._session = Session(**session_args)
self._client = self._session.client(self._resource_name, config=self._config)
self._resource = self._session.resource(self._resource_name, config=self._config)
return self
def __exit__(self, type, value, traceback):
"""Callback handles destroying an existing client."""
pass
@staticmethod
def instantiate_credentials_from_config(config):
if 'credentials' in config and config['credentials']:
credentials_dict = config['credentials']
if 'access_key_id' not in credentials_dict or not credentials_dict['access_key_id']:
raise InvalidAWSCredentials('"credentials" requires "access_key_id" to be populated')
if 'secret_access_key' not in credentials_dict or not credentials_dict['secret_access_key']:
raise InvalidAWSCredentials('"credentials" requires "secret_access_key" to be populated')
return AWSCredentials(credentials_dict['access_key_id'], credentials_dict['secret_access_key'])
def test_create_security_group(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
sg_ctx = {
'name': 'sg01a',
'description': 'Test security group sg01a',
'vpc': 'vpc01'
}
tags = [
{
'Key': 'Name',
'Value': 'sg01a'
}
]
vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
sg_filters = [{'Name': 'tag:Name', 'Values': ['sg01a']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_sg.SecurityGroupWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.SecurityGroup'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=vpc_filters))
vpc = vpcs[0]
# Create the security group
h = ec2_sg.create_handler(sg_ctx, self.credentials)
h.create_resource()
security_groups = list(ec2.security_groups.filter(Filters=sg_filters))
sg = security_groups[0]
self.assertEqual(len(security_groups), 1)
self.assertEqual(sg.name, 'sg01a')
# Security groups have a dedicated attribute for their name
self.assertEqual(sg.name, sg.group_name)
self.assertEqual(sg.vpc_id, vpc.id)
self.assertCountEqual(sg.tags, tags)
def get_table(
table_name: str,
profile: Optional[str] = None,
region: Optional[str] = None,
retries: Optional[int] = None,
):
from boto3 import Session
from botocore.config import Config
session = Session(profile_name=profile, region_name=region)
config = Config(retries={"max_attempts": retries}) if retries else Config()
dynamodb = session.resource("dynamodb", config=config)
return dynamodb.Table(table_name)
def __init__(self,
table_name,
aws_access_key_id=None,
aws_secret_access_key=None,
aws_region_name='us-west-2'):
if aws_access_key_id and aws_secret_access_key:
session = Session(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
region_name=aws_region_name)
self.resource = session.resource('dynamodb')
else:
self.resource = boto3.resource('dynamodb')
self.table = self.resource.Table(table_name)
class Resource_Dynamodb :
def __init__(self,iamProfilename,tableName) :
from boto3 import Session
self.session = Session(profile_name=iamProfilename)
self.dynamodb = self.session.resource('dynamodb')
self.table = self.dynamodb.Table(tableName)
def _getCountOfQueryResult(self, partitionKey, condiTion):
from boto3.dynamodb.conditions import Key
# Reference1 : https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Python.04.html
# Reference2 : http://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.Client.query
response = self.table.query(
KeyConditionExpression = Key(partitionKey).eq(condiTion),
Select = 'COUNT'
)
return(response)
def upload_file_by_file(file):
milli_sec = str(datetime.datetime.now())
filename = str(milli_sec) + '.pdf'
print(file)
session = Session(aws_access_key_id=settings.AWS_ACCESS_KEY,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=settings.AWS_REGION_NAME)
s3 = session.resource('s3')
file_key = FileUploadUtils.getFileKey()
res = s3.Bucket(settings.AWS_BUCKET_NAME).put_object(Key=file_key, Body=file, ContentType='application/pdf',
ACL='public-read')
data = {'key': file_key, 'file_name': filename, 'is_link': False}
serializer = serializers.CreateFileUploadSerializer(data=data)
if serializer.is_valid():
serializer.save()
media = Media.objects.get(key=file_key)
return media
class Accessor():
def __init__(self, bucket_name, profile_name=None):
"""
s3との接続を確立する
"""
self.profile_name = profile_name
self.bucket_name = bucket_name
self.session = Session(profile_name=profile_name)
self.s3 = self.session.resource('s3')
self.bucket = self.s3.Bucket(bucket_name)
def upload_file(self, src, dst):
"""
src:アップロードするファイルのパス
dst:アップロード先(ファイル名含む)
例：src=/tmp/hoge.tsv, dst=HOGE/FUGA/hoge.tsv
設定した bucketの HOGE/FUGAにhoge.tsv という名前でアップロードする。ディレクトリは自動で作成される
"""
self.bucket.upload_file(src, dst)
def download_file(self, src, dst):
"""
src:ダウンロードするファイルのパス
dst:ダウンロード先(ファイル名含む)
例：src=HOGE/FUGA/hoge.tsv, dst=/hoge/fuga.tsv
設定した bucketの HOGE/FUGAにhoge.tsv を /hoge/fuga.tsv という名前でダウンロードする
"""
if not os.path.exists(os.path.dirname(dst)):
os.makedirs(os.path.dirname(dst))
self.bucket.download_file(src, dst)
def download_directory(self, src_dir, dst_dir):
"""
指定したディレクトリ内のファイルをダウンロードする
src_dir:ダウンロードするディレクトリ
dst_dir:ダウンロード先ディレクトリ
例：src=HOGE/FUGA, dst=/hoge
設定した bucketの HOGE/FUGA 内のファイルを を /hoge 以下にダウンロードする
"""
total = len(list(self.bucket.objects.filter(Prefix=src_dir)))
for obj in tqdm(self.bucket.objects.filter(Prefix=src_dir),
total=total):
logname = os.path.basename(obj.key)
dst = os.path.join(dst_dir, logname)
self.download_file(obj.key, dst)
def test_delete_security_group(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
sg_ctx = {
'name': 'sg01a',
'description': 'Test security group sg01a',
'vpc': 'vpc01'
}
filters = [{'Name': 'tag:Name', 'Values': ['sg01a']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_sg.SecurityGroupWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.SecurityGroup'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the security group
h = ec2_sg.create_handler(sg_ctx, self.credentials)
h.create_resource()
security_groups = list(ec2.security_groups.filter(Filters=filters))
self.assertEqual(len(security_groups), 1)
# We clear the resource cache to simulate a new
# program execution with the 'delete' option
base.BaseHandler._cache.clear()
# Delete the security group
h.delete_resource()
security_groups = list(ec2.security_groups.filter(Filters=filters))
self.assertEqual(len(security_groups), 0)
def test_attach_internet_gateway(self):
igw_ctx = {
'name': 'igw01'
}
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'internet_gateway': 'igw01'
}
igw_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}]
vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_vpc.VpcWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Vpc'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the internet gateway
h = ec2_igw.create_handler(igw_ctx, self.credentials)
h.create_resource()
gateways = list(ec2.internet_gateways.filter(Filters=igw_filters))
igw = gateways[0]
self.assertCountEqual(igw.attachments, [])
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=vpc_filters))
vpc = vpcs[0]
# Test that the internet gateway has been attached
igw.reload()
attachments = [{'VpcId': vpc.id, 'State': 'available'}]
self.assertCountEqual(igw.attachments, attachments)
def deleteFile(key):
media = Media.objects.get(id=key)
session = Session(aws_access_key_id=settings.AWS_ACCESS_KEY,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
region_name=settings.AWS_REGION_NAME)
s3 = session.resource('s3')
my_bucket = s3.Bucket(settings.AWS_BUCKET_NAME)
response = my_bucket.delete_objects(
Delete={
'Objects': [
{
'Key': media.key
}
]
}
)
media.delete()
return response
def test_create_vpc(self):
ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'tags': {
'description': 'VPC vpc01 (subnet01a & subnet01b)'
}
}
tags = [
{
'Key': 'Name',
'Value': 'vpc01'
},
{
'Key': 'Description',
'Value': 'VPC vpc01 (subnet01a & subnet01b)'
}
]
filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_vpc.VpcWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Vpc'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(ctx, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=filters))
vpc = vpcs[0]
self.assertEqual(len(vpcs), 1)
self.assertEqual(vpc.name, 'vpc01')
self.assertEqual(vpc.cidr_block, '10.0.10.0/24')
self.assertCountEqual(vpc.tags, tags)
def test_create_internet_gateway(self):
ctx = {
'name': 'igw01',
'tags': {
'description': 'Internet gateway for VPC vpc01'
}
}
tags = [
{
'Key': 'Name',
'Value': 'igw01'
},
{
'Key': 'Description',
'Value': 'Internet gateway for VPC vpc01'
}
]
filters = [{'Name': 'tag:Name', 'Values': ['igw01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_igw.InternetGatewayWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.InternetGateway'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the internet gateway
h = ec2_igw.create_handler(ctx, self.credentials)
h.create_resource()
gateways = list(ec2.internet_gateways.filter(Filters=filters))
igw = gateways[0]
self.assertEqual(len(gateways), 1)
self.assertEqual(igw.name, 'igw01')
self.assertCountEqual(igw.tags, tags)
self.assertCountEqual(igw.attachments, [])
def test_delete_dhcp_options_set(self):
ctx = {
'name': 'dhcp01',
'domain_name': [
'test01.us-west-2.aws'
],
'domain_name_servers': [
'10.0.10.2'
]
}
filters = [{'Name': 'tag:Name', 'Values': ['dhcp01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_dhcp.DhcpOptionsWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.DhcpOptions'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the DHCP options set
h = ec2_dhcp.create_handler(ctx, self.credentials)
h.create_resource()
dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters))
self.assertEqual(len(dhcp_options_sets), 1)
# We clear the resource cache to simulate a new
# program execution with the 'delete' option
base.BaseHandler._cache.clear()
# Delete the DHCP options set
h.delete_resource()
dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters))
self.assertEqual(len(dhcp_options_sets), 0)
class Website:
def __init__(self, bucket_name, file_location, access_key, secret_access_key, region):
self.bucket_name = bucket_name
self.file_location = file_location
self.session = Session(
aws_access_key_id=access_key, aws_secret_access_key=secret_access_key, region_name=region
)
self.s3 = self.session.resource("s3")
self.bucket = self.s3.Bucket(bucket_name)
def upload_files_to_amazon(self):
for root, directory, files in os.walk(self.file_location):
for f in files:
file_location = os.path.abspath(os.path.join(root, f))
file_key = file_location.replace(self.file_location, "")
# Make sure the documents are correctly presented when they get on the server.
extra_args = dict()
if f.endswith(".css"):
extra_args["ContentType"] = "text/css"
elif f.endswith(".json"):
extra_args["ContentType"] = "application/json"
elif f.endswith(".js"):
extra_args["ContentType"] = "application/javascript"
elif f.endswith(".html"):
extra_args["ContentType"] = "text/html"
elif f.endswith(".txt"):
extra_args["ContentType"] = "text/plain"
elif f.endswith(".pdf"):
extra_args["ContentType"] = "application/pdf"
elif f.endswith(".jpeg"):
extra_args["ContentType"] = "image/jpeg"
elif f.endswith(".png"):
extra_args["ContentType"] = "image/png"
elif f.endswith(".bmp"):
extra_args["ContentType"] = "image/bmp"
elif f.endswith(".gif"):
extra_args["ContentType"] = "image/gif"
self.s3.meta.client.upload_file(file_location, self.bucket_name, file_key, extra_args)
def test_revoke_egress_rule(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
sg_ctx1 = {
'name': 'sg01a',
'description': 'Test security group sg01a',
'vpc': 'vpc01',
'egress': [
{
'ip_protocol': '-1',
'from_port': -1,
'to_port': -1,
'destinations': [
'0.0.0.0/0'
]
}
]
}
sg_ctx2 = {
'name': 'sg01a',
'description': 'Test security group sg01a',
'vpc': 'vpc01'
}
ip_permission = {
'IpProtocol': '-1',
'FromPort': -1,
'ToPort': -1,
'IpRanges': [
{
'CidrIp': '0.0.0.0/0'
}
],
'UserIdGroupPairs': []
}
rule = ec2_sg.RuleWrapper(ip_permission, flow='destination')
filters = [{'Name': 'tag:Name', 'Values': ['sg01a']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_sg.SecurityGroupWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.SecurityGroup'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the security group
h = ec2_sg.create_handler(sg_ctx1, self.credentials)
h.create_resource()
security_groups = list(ec2.security_groups.filter(Filters=filters))
sg = security_groups[0]
self.assertEqual(len(security_groups), 1)
self.assertEqual(len(sg.egress_rules), 1)
self.assertIn(rule, sg.egress_rules)
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the security group
h = ec2_sg.create_handler(sg_ctx2, self.credentials)
h.update_resource()
security_groups = list(ec2.security_groups.filter(Filters=filters))
sg = security_groups[0]
self.assertEqual(len(security_groups), 1)
self.assertEqual(len(sg.egress_rules), 0)
class InternetGatewayHandler(base.BaseHandler):
"""Manage ``EC2.InternetGateway`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra features
# to the base class EC2.InternetGateway
event = "creating-resource-class.ec2.InternetGateway"
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource("ec2")
self._client = self._session.client("ec2")
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get("name")
def create_resource(self):
"""Create an internet gateway, or update it if it already exists."""
self._resource = self._load(self._name, manager_name="internet_gateways")
if self._resource:
logger.info("Internet gateway %s already exists.", self._name)
else:
self._create_internet_gateway()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update an internet gateway object."""
self._resource = self._load(self._name, manager_name="internet_gateways")
if self._resource:
self._update_internet_gateway()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error("Internet gateway %s does not exist.", self._name)
def delete_resource(self):
"""Delete an ``InternetGateway`` object."""
self._resource = self._load(self._name, manager_name="internet_gateways")
if self._resource:
logger.info("Removing internet gateway %s.", self._name)
if self._resource.attachments:
# If the internet gateway is attached to a VPC,
# we don't take the responsibility to detach it
vpc_id = self._resource.attachments[0]["VpcId"]
logger.warning(
"The internet gateway %s is currently attached to VPC %s." " Operation not performed.",
self._name,
vpc_id,
)
else:
try:
self._client.delete_internet_gateway(InternetGatewayId=self._resource.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error("Internet gateway %s does not exist.", self._name)
def _create_internet_gateway(self):
"""Create an ``InternetGateway`` object."""
logger.info("Creating internet gateway %s.", self._name)
try:
self._resource = self._ec2.create_internet_gateway()
except ClientError as exc:
logger.error(exc)
return
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_internet_gateway()
def _update_internet_gateway(self):
"""Update a ``InternetGateway`` object."""
self._update_tags()
def test_update_security_group(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
sg_ctx1 = {
'name': 'sg01b',
'description': 'Test security group sg01b',
'vpc': 'vpc01',
'tags': {
'stack': 'Test',
'owner': 'Team A'
}
}
sg_ctx2 = {
'name': 'sg01b',
'description': 'Test security group sg01b',
'vpc': 'vpc01',
'tags': {
'stack': 'Production',
'platform': 'app01'
}
}
tags1 = [
{
'Key': 'Name',
'Value': 'sg01b'
},
{
'Key': 'Stack',
'Value': 'Test'
},
{
'Key': 'Owner',
'Value': 'Team A'
}
]
tags2 = [
{
'Key': 'Name',
'Value': 'sg01b'
},
{
'Key': 'Stack',
'Value': 'Production'
},
{
'Key': 'Platform',
'Value': 'app01'
}
]
sg_filters = [{'Name': 'tag:Name', 'Values': ['sg01b']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_sg.SecurityGroupWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.SecurityGroup'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the security group
h = ec2_sg.create_handler(sg_ctx1, self.credentials)
h.create_resource()
security_groups = list(ec2.security_groups.filter(Filters=sg_filters))
sg = security_groups[0]
sg_id = sg.id
self.assertEqual(len(security_groups), 1)
self.assertCountEqual(sg.tags, tags1)
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the security group
h = ec2_sg.create_handler(sg_ctx2, self.credentials)
h.update_resource()
security_groups = list(ec2.security_groups.filter(Filters=sg_filters))
sg = security_groups[0]
self.assertEqual(len(security_groups), 1)
self.assertEqual(sg.id, sg_id)
self.assertCountEqual(sg.tags, tags2)
def test_update_subnet(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
subnet_ctx1 = {
'name': 'subnet01b',
'cidr_block': '10.0.10.128/25',
'zone': 'us-west-2b',
'vpc': 'vpc01',
'tags': {
'description': 'Test subnet (zone b) for VPC vpc01',
'stack': 'Test',
'owner': 'Team A'
}
}
subnet_ctx2 = {
'name': 'subnet01b',
'cidr_block': '10.0.10.128/25',
'zone': 'us-west-2b',
'vpc': 'vpc01',
'tags': {
'description': 'Test subnet (zone b) for VPC vpc01',
'stack': 'Production'
}
}
tags1 = [
{
'Key': 'Name',
'Value': 'subnet01b'
},
{
'Key': 'Description',
'Value': 'Test subnet (zone b) for VPC vpc01'
},
{
'Key': 'Stack',
'Value': 'Test'
},
{
'Key': 'Owner',
'Value': 'Team A'
}
]
tags2 = [
{
'Key': 'Name',
'Value': 'subnet01b'
},
{
'Key': 'Description',
'Value': 'Test subnet (zone b) for VPC vpc01'
},
{
'Key': 'Stack',
'Value': 'Production'
}
]
subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01b']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_subnet.SubnetWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Subnet'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the subnet
h = ec2_subnet.create_handler(subnet_ctx1, self.credentials)
h.create_resource()
subnets = list(ec2.subnets.filter(Filters=subnet_filters))
subnet = subnets[0]
subnet_id = subnet.id
self.assertEqual(len(subnets), 1)
self.assertCountEqual(subnet.tags, tags1)
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the subnet
h = ec2_subnet.create_handler(subnet_ctx2, self.credentials)
h.update_resource()
subnets = list(ec2.subnets.filter(Filters=subnet_filters))
subnet = subnets[0]
self.assertEqual(len(subnets), 1)
self.assertEqual(subnet.id, subnet_id)
self.assertCountEqual(subnet.tags, tags2)
from __future__ import print_function
from HTMLParser import HTMLParser
import requests
from boto3 import Session
from boto3.dynamodb.conditions import Key, Attr
import uuid,time
session = Session(region_name='eu-west-1')
dynamodb = session.resource('dynamodb')
table = dynamodb.Table('GameMetrics')
def lambda_handler(event, context):
class reviews(HTMLParser):
def __init__(self):
HTMLParser.__init__(self)
self.inLink = False
self.dataArray = []
self.countLanguages = 0
self.lasttag = None
self.lastname = None
self.lastvalue = None
def handle_starttag(self, tag, attrs):
self.inLink = False
if tag == 'span':
for name, value in attrs:
if name == 'class' and value =='user_reviews_count':
self.countLanguages += 1
def test_update_map_public_ip_on_launch(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
subnet_ctx1 = {
'name': 'subnet01b',
'cidr_block': '10.0.10.128/25',
'zone': 'us-west-2b',
'vpc': 'vpc01'
}
subnet_ctx2 = {
'name': 'subnet01b',
'cidr_block': '10.0.10.128/25',
'zone': 'us-west-2b',
'vpc': 'vpc01',
'map_public_ip_on_launch': True
}
subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01b']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_subnet.SubnetWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Subnet'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the subnet
h = ec2_subnet.create_handler(subnet_ctx1, self.credentials)
h.create_resource()
subnets = list(ec2.subnets.filter(Filters=subnet_filters))
subnet = subnets[0]
subnet_id = subnet.id
self.assertEqual(len(subnets), 1)
self.assertEqual(subnet.name, 'subnet01b')
self.assertEqual(subnet.cidr_block, '10.0.10.128/25')
self.assertEqual(subnet.availability_zone, 'us-west-2b')
self.assertEqual(subnet.map_public_ip_on_launch, False)
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the subnet
h = ec2_subnet.create_handler(subnet_ctx2, self.credentials)
h.update_resource()
subnets = list(ec2.subnets.filter(Filters=subnet_filters))
subnet = subnets[0]
self.assertEqual(len(subnets), 1)
self.assertEqual(subnet.id, subnet_id)
self.assertEqual(subnet.map_public_ip_on_launch, True)
def test_delete_route(self):
igw_ctx = {
'name': 'igw01'
}
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'internet_gateway': 'igw01'
}
rt_ctx1 = {
'name': 'rt01',
'vpc': 'vpc01',
'routes': [
{
'destination': '0.0.0.0/0',
'target_name': 'igw01',
'target_type': 'internet_gateway'
}
]
}
rt_ctx2 = {
'name': 'rt01',
'vpc': 'vpc01',
'routes': None
}
igw_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}]
rt_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_rt.RouteTableWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.RouteTable'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the internet gateway
h = ec2_igw.create_handler(igw_ctx, self.credentials)
h.create_resource()
internet_gateways = list(ec2.internet_gateways.filter(Filters=igw_filters))
igw = internet_gateways[0]
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the route table
h = ec2_rt.create_handler(rt_ctx1, self.credentials)
h.create_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt_filters))
rt = route_tables[0]
route = {
'GatewayId': igw.id,
'DestinationCidrBlock': '0.0.0.0/0',
'Origin': 'CreateRoute',
'State': 'active'
}
route_wrapper = ec2_rt.RouteWrapper(route)
self.assertEqual(len(rt.routes), 2) # the new route + the default VPC route
self.assertIn(route_wrapper, rt.custom_routes)
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the route table
h = ec2_rt.create_handler(rt_ctx2, self.credentials)
h.update_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt_filters))
rt = route_tables[0]
self.assertEqual(len(rt.routes), 1) # the default VPC route
self.assertCountEqual(rt.custom_routes, [])
def test_update_route_table(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
rt_ctx1 = {
'name': 'rt01',
'vpc': 'vpc01',
'tags': {
'description': 'Replace the default route table for VPC vpc01',
'stack': 'Test',
'owner': 'Team A'
}
}
rt_ctx2 = {
'name': 'rt01',
'vpc': 'vpc01',
'tags': {
'description': 'Replace the default route table for VPC vpc01',
'stack': 'Production'
}
}
tags1 = [
{
'Key': 'Name',
'Value': 'rt01'
},
{
'Key': 'Description',
'Value': 'Replace the default route table for VPC vpc01'
},
{
'Key': 'Stack',
'Value': 'Test'
},
{
'Key': 'Owner',
'Value': 'Team A'
}
]
tags2 = [
{
'Key': 'Name',
'Value': 'rt01'
},
{
'Key': 'Description',
'Value': 'Replace the default route table for VPC vpc01'
},
{
'Key': 'Stack',
'Value': 'Production'
}
]
rt_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_rt.RouteTableWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.RouteTable'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the route table
h = ec2_rt.create_handler(rt_ctx1, self.credentials)
h.create_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt_filters))
rt = route_tables[0]
rt_id = rt.id
self.assertEqual(len(route_tables), 1)
self.assertCountEqual(rt.tags, tags1)
# Update the route table
h = ec2_rt.create_handler(rt_ctx2, self.credentials)
h.update_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt_filters))
rt = route_tables[0]
self.assertEqual(len(route_tables), 1)
self.assertEqual(rt.id, rt_id)
self.assertCountEqual(rt.tags, tags2)
class SecurityGroupHandler(BaseHandler):
"""Manage ``EC2.SecurityGroup`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra
# features to the base class EC2.SecurityGroup
event = 'creating-resource-class.ec2.SecurityGroup'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a security group, or update it if it already exists."""
self._resource = self._load(self._name, manager_name='security_groups')
if self._resource:
logger.info('Security group %s already exists.', self._name)
else:
self._create_security_group()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a security group object."""
self._resource = self._load(self._name, manager_name='security_groups')
if self._resource:
self._update_security_group()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('Security group %s does not exist.', self._name)
@retry(**retry_params)
def delete_resource(self):
"""Delete a ``SecurityGroup`` object."""
self._resource = self._load(self._name, manager_name='security_groups')
if self._resource:
logger.info('Removing security group %s.', self._name)
try:
self._resource.delete(GroupId=self._resource.id)
except ClientError as exc:
# This is intended to prevent potential errors when trying
# to remove security groups that may depend on each other.
# If a 'DependencyViolation' is raised, we try to delete
# the security group again after a short delay.
if exc.response['Error']['Code'] == 'DependencyViolation':
logger.info(
'Security group %s has a dependency and can '
'not be removed. Trying again...', self._name
)
raise DependencyViolationError
else:
logger.error(exc)
else:
logger.error('Security group %s does not exist.', self._name)
def _create_security_group(self):
"""Create a ``SecurityGroup`` object."""
vpc_name = self._ctx['vpc']
vpc = self._load(vpc_name, manager_name='vpcs')
if vpc:
logger.info('Creating security group %s.', self._name)
try:
self._resource = self._ec2.create_security_group(
GroupName=self._name,
Description=self._ctx['description'],
VpcId=vpc.id
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
return
# Remove the default outbound rule authorizing all outgoing traffic
ip_permission = {
'IpProtocol': '-1',
'FromPort': -1,
'ToPort': -1,
'IpRanges': [{'CidrIp': '0.0.0.0/0'}],
'UserIdGroupPairs': []
}
logger.info('%s: revoking default outbound rule.', self._name)
try:
self._resource.revoke_egress(IpPermissions=[ip_permission])
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_security_group()
else:
logger.error(
'Unable to create security group %s because VPC %s does not'
' exist. Operation not performed.', self._name, vpc_name
)
def _update_security_group(self):
"""Update a ``SecurityGroup`` object."""
self._update_tags()
self._update_ingress_rules()
self._update_egress_rules()
def _update_ingress_rules(self):
"""Manage security group ingress rules."""
current_ingress_rules = self._resource.ingress_rules
new_ingress_rules = []
to_authorize = []
to_revoke = []
for rule in self._ctx.get('ingress', []):
for source in rule['sources']:
ip_permission = {
'IpProtocol': str(rule['ip_protocol']),
'FromPort': int(rule['from_port']),
'ToPort': int(rule['to_port'])
}
try:
ipaddress.ip_network(source)
# 'source' attribute is a valid CIDR
ip_permission['IpRanges'] = [{'CidrIp': source}]
ip_permission['UserIdGroupPairs'] = []
except ValueError:
# Assuming we got a security group ID
sg = self._load(source, manager_name='security_groups')
if sg:
pair = {'GroupId': sg.id, 'UserId': sg.owner_id}
ip_permission['UserIdGroupPairs'] = [pair]
ip_permission['IpRanges'] = []
else:
logger.error(
"%s: invalid value (source security group "
"'%s' does not exist).", self._name, source
)
continue
new_ingress_rules.append(
RuleWrapper(ip_permission, flow='source')
)
for rule in set(new_ingress_rules) - set(current_ingress_rules):
logger.info('%s: authorizing inbound rule %s.', self._name, rule)
to_authorize.append(rule.ip_permission)
for rule in set(current_ingress_rules) - set(new_ingress_rules):
logger.info('%s: revoking inbound rule %s.', self._name, rule)
to_revoke.append(rule.ip_permission)
try:
if to_authorize:
self._resource.authorize_ingress(IpPermissions=to_authorize)
if to_revoke:
self._resource.revoke_ingress(IpPermissions=to_revoke)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
def _update_egress_rules(self):
"""Manage security group egress rules."""
current_egress_rules = self._resource.egress_rules
new_egress_rules = []
to_authorize = []
to_revoke = []
for rule in self._ctx.get('egress', []):
for destination in rule['destinations']:
ip_permission = {
'IpProtocol': str(rule['ip_protocol']),
'FromPort': int(rule['from_port']),
'ToPort': int(rule['to_port'])
}
try:
ipaddress.ip_network(destination)
# 'destination' attribute is a valid CIDR
ip_permission['IpRanges'] = [{'CidrIp': destination}]
ip_permission['UserIdGroupPairs'] = []
except ValueError:
# Assuming we got a security group ID
sg = self._load(
destination, manager_name='security_groups'
)
if sg:
pair = {'GroupId': sg.id, 'UserId': sg.owner_id}
ip_permission['UserIdGroupPairs'] = [pair]
ip_permission['IpRanges'] = []
else:
logger.error(
"%s: invalid value (destination security group "
"'%s' does not exist).", self._name, destination
)
continue
new_egress_rules.append(
RuleWrapper(ip_permission, flow='destination')
)
for rule in set(new_egress_rules) - set(current_egress_rules):
logger.info('%s: authorizing outbound rule %s.', self._name, rule)
to_authorize.append(rule.ip_permission)
for rule in set(current_egress_rules) - set(new_egress_rules):
logger.info('%s: revoking outbound rule %s.', self._name, rule)
to_revoke.append(rule.ip_permission)
try:
if to_authorize:
self._resource.authorize_egress(IpPermissions=to_authorize)
if to_revoke:
self._resource.revoke_egress(IpPermissions=to_revoke)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
def test_change_subnet_association(self):
vpc_ctx = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24'
}
subnet_ctx = {
'name': 'subnet01a',
'cidr_block': '10.0.10.0/25',
'zone': 'us-west-2a',
'vpc': 'vpc01'
}
rt_ctx1 = {
'name': 'rt01',
'vpc': 'vpc01',
'subnets': [
'subnet01a'
]
}
rt_ctx2 = {
'name': 'rt02',
'vpc': 'vpc01',
'subnets': [
'subnet01a'
]
}
subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01a']}]
rt01_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}]
rt02_filters = [{'Name': 'tag:Name', 'Values': ['rt02']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_rt.RouteTableWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.RouteTable'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(vpc_ctx, self.credentials)
h.create_resource()
# Create the subnet
h = ec2_subnet.create_handler(subnet_ctx, self.credentials)
h.create_resource()
subnets = list(ec2.subnets.filter(Filters=subnet_filters))
subnet = subnets[0]
# Create route table rt01
h = ec2_rt.create_handler(rt_ctx1, self.credentials)
h.create_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt01_filters))
rt01 = route_tables[0]
filters = [
{
'Name': 'association.subnet-id',
'Values': [subnet.id]
},
{
'Name': 'association.route-table-id',
'Values': [rt01.id]
}
]
associations = list(rt01.associations.filter(Filters=filters))
self.assertEqual(len(associations), 1)
# Create route table rt02
h = ec2_rt.create_handler(rt_ctx2, self.credentials)
h.create_resource()
route_tables = list(ec2.route_tables.filter(Filters=rt02_filters))
rt02 = route_tables[0]
filters = [
{
'Name': 'association.subnet-id',
'Values': [subnet.id]
},
{
'Name': 'association.route-table-id',
'Values': [rt02.id]
}
]
associations = list(rt02.associations.filter(Filters=filters))
self.assertEqual(len(associations), 1)
class DhcpOptionsHandler(base.BaseHandler):
"""Manage ``EC2.DhcpOptions`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra features
# to the base class EC2.DhcpOptions
event = 'creating-resource-class.ec2.DhcpOptions'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a DHCP options set, or update it if it already exists."""
self._resource = self._load(
self._name, manager_name='dhcp_options_sets'
)
if self._resource:
logger.info('DHCP options set %s already exists.', self._name)
else:
self._create_dhcp_options()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a ``DhcpOptions`` object."""
self._resource = self._load(
self._name, manager_name='dhcp_options_sets'
)
if self._resource:
self._update_dhcp_options()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('DHCP options set %s does not exist.', self._name)
def delete_resource(self):
"""Delete a ``DhcpOptions`` object."""
self._resource = self._load(
self._name, manager_name='dhcp_options_sets'
)
if self._resource:
logger.info('Removing DHCP options set %s.', self._name)
try:
self._client.delete_dhcp_options(
DhcpOptionsId=self._resource.id
)
except ClientError as exc:
# If the DHCP options set is associated with a
# VPC, we just log the error and ignore it, as
# the options set object does not provide any
# method to detach itself from the VPC
logger.error(exc)
else:
logger.error('DHCP options set %s does not exist.', self._name)
def _create_dhcp_options(self):
"""Create a ``DhcpOptions`` object."""
dhcp_configurations = []
attrs = ['domain_name_servers',
'domain_name',
'ntp_servers',
'netbios_name_servers',
'netbios_node_type']
for attr in attrs:
if attr in self._ctx:
dhcp_configurations.append({
'Key': attr.replace('_', '-'),
'Values': self._ctx.get(attr, [])
})
logger.info('Creating DHCP options set %s.', self._name)
try:
self._resource = self._ec2.create_dhcp_options(
DhcpConfigurations=dhcp_configurations
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
return
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_dhcp_options()
def _update_dhcp_options(self):
"""Update a ``DhcpOptions`` object."""
self._update_tags()
def __init__(self, instance_id: str):
session = Session()
self.__instance = session.resource('ec2').Instance(instance_id)
self.__ssm = session.client('ssm')
def test_another_internet_gateway_already_attached_2(self):
# Case 2: an internet gateway is attached to the VPC and there
# is another internet gateway defined in the configuration. We
# also leave everything unchanged.
igw01_ctx = {
'name': 'igw01'
}
igw02_ctx = {
'name': 'igw02'
}
vpc01_ctx1 = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'internet_gateway': 'igw01'
}
vpc01_ctx2 = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'internet_gateway': 'igw02'
}
igw01_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}]
igw02_filters = [{'Name': 'tag:Name', 'Values': ['igw02']}]
vpc01_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_vpc.VpcWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Vpc'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the first internet gateway
h = ec2_igw.create_handler(igw01_ctx, self.credentials)
h.create_resource()
gateways = list(ec2.internet_gateways.filter(Filters=igw01_filters))
igw01 = gateways[0]
self.assertCountEqual(igw01.attachments, [])
# Create the VPC
h = ec2_vpc.create_handler(vpc01_ctx1, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=vpc01_filters))
vpc01 = vpcs[0]
attachments = [{'VpcId': vpc01.id, 'State': 'available'}]
# Test that the internet gateway has been attached
igw01.reload()
self.assertCountEqual(igw01.attachments, attachments)
# Create the second internet gateway
h = ec2_igw.create_handler(igw02_ctx, self.credentials)
h.create_resource()
gateways = list(ec2.internet_gateways.filter(Filters=igw02_filters))
igw02 = gateways[0]
self.assertCountEqual(igw02.attachments, [])
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the VPC with the new context
h = ec2_vpc.create_handler(vpc01_ctx2, self.credentials)
h.update_resource()
# Test that the first internet gateway is still attached to the VPC
igw01.reload()
self.assertCountEqual(igw01.attachments, attachments)
# Test that the second internet gateway is not attached
igw02.reload()
self.assertCountEqual(igw02.attachments, [])
def test_update_vpc(self):
ctx1 = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'tags': {
'description': 'VPC vpc01 (subnet01a & subnet01b)',
'stack': 'Test',
'owner': 'Team A'
}
}
ctx2 = {
'name': 'vpc01',
'cidr_block': '10.0.10.0/24',
'tags': {
'description': 'VPC vpc01 (subnet01a & subnet01b)',
'stack': 'Production'
}
}
tags1 = [
{
'Key': 'Name',
'Value': 'vpc01'
},
{
'Key': 'Description',
'Value': 'VPC vpc01 (subnet01a & subnet01b)'
},
{
'Key': 'Stack',
'Value': 'Test'
},
{
'Key': 'Owner',
'Value': 'Team A'
}
]
tags2 = [
{
'Key': 'Name',
'Value': 'vpc01'
},
{
'Key': 'Description',
'Value': 'VPC vpc01 (subnet01a & subnet01b)'
},
{
'Key': 'Stack',
'Value': 'Production'
}
]
filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}]
def _add_wrapper(base_classes, **kwargs):
base_classes.insert(0, ec2_vpc.VpcWrapper)
with mock_ec2():
event = 'creating-resource-class.ec2.Vpc'
session = Session(**self.credentials)
session.events.register(event, _add_wrapper)
ec2 = session.resource('ec2')
# Create the VPC
h = ec2_vpc.create_handler(ctx1, self.credentials)
h.create_resource()
vpcs = list(ec2.vpcs.filter(Filters=filters))
vpc = vpcs[0]
vpc_id = vpc.id
self.assertEqual(len(vpcs), 1)
self.assertCountEqual(vpc.tags, tags1)
# We clear the resource cache to simulate a new
# program execution with the 'update' option
base.BaseHandler._cache.clear()
# Update the VPC
h = ec2_vpc.create_handler(ctx2, self.credentials)
h.update_resource()
vpcs = list(ec2.vpcs.filter(Filters=filters))
vpc = vpcs[0]
self.assertEqual(len(vpcs), 1)
self.assertEqual(vpc.id, vpc_id)
self.assertCountEqual(vpc.tags, tags2)
class RouteTableHandler(base.BaseHandler):
"""Manage ``EC2.RouteTable`` object lifecycle."""
def __init__(self, ctx, credentials):
"""Create boto3 session and clients."""
super().__init__()
# Create a boto session and add extra
# features to the base class EC2.RouteTable
event = 'creating-resource-class.ec2.RouteTable'
self._session = Session(**credentials)
self._session.events.register(event, _add_wrapper)
# boto3 clients
self._ec2 = self._session.resource('ec2')
self._client = self._session.client('ec2')
# Context for the resource we want
# to create, update or delete
self._ctx = ctx
self._name = ctx.get('name')
def create_resource(self):
"""Create a route table, or update it if it already exists."""
self._resource = self._load(self._name, manager_name='route_tables')
if self._resource:
logger.info('Route table %s already exists.', self._name)
else:
self._create_route_table()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
def update_resource(self):
"""Update a route table object."""
self._resource = self._load(self._name, manager_name='route_tables')
if self._resource:
self._update_route_table()
# Store the object in the cache for future use
with self._lock:
self._cache[self._name] = self._resource
else:
logger.error('Route table %s does not exist.', self._name)
def delete_resource(self):
"""Delete a ``RouteTable`` object."""
self._resource = self._load(self._name, manager_name='route_tables')
if self._resource:
# Remove subnet associations before deleting the route table
for subnet_name in self._ctx.get('subnets') or []:
subnet = self._load(subnet_name, manager_name='subnets')
if subnet:
filters = [
{
'Name': 'association.subnet-id',
'Values': [subnet.id]
},
{
'Name': 'association.route-table-id',
'Values': [self._resource.id]
}
]
associations = list(self._resource.associations.filter(Filters=filters))
if associations:
# The subnet is associated with this route table
try:
associations[0].delete()
except ClientError as exc:
logger.error(exc)
# Remove the route table
logger.info('Removing route table %s.', self._name)
try:
self._client.delete_route_table(RouteTableId=self._resource.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error('Route table %s does not exist.', self._name)
def _create_route_table(self):
"""Create a ``RouteTable`` object."""
vpc_name = self._ctx['vpc']
vpc = self._load(vpc_name, manager_name='vpcs')
if vpc:
logger.info('Creating route table %s.', self._name)
try:
self._resource = self._ec2.create_route_table(VpcId=vpc.id)
except ClientError as exc:
logger.error(exc)
return
# Set the value of the 'Name' tag
self._resource.name = self._name
# Update other object attributes
self._update_route_table()
else:
logger.error(
'Unable to create route table %s because VPC %s does not'
' exist. Operation not performed.', self._name, vpc_name
)
def _update_route_table(self):
"""Update a ``RouteTable`` object."""
self._update_tags()
self._update_subnets()
self._update_routes()
def _update_subnets(self):
"""Manage subnet associations for the current route table."""
for subnet_name in self._ctx.get('subnets') or []:
subnet = self._load(subnet_name, manager_name='subnets')
if subnet:
# Search if this subnet is already associated with a route table
filters = [
{
'Name': 'association.subnet-id',
'Values': [subnet.id]
}
]
route_tables = list(self._ec2.route_tables.filter(Filters=filters))
if route_tables:
if route_tables[0].id == self._resource.id:
# The subnet is already associated with this route table
continue
else:
# The subnet is associated with another route table
# and we need to update the association
associations = list(route_tables[0].associations.filter(Filters=filters))
try:
associations[0].replace_subnet(RouteTableId=self._resource.id)
logger.info(
'Subnet %s is now associated with route table %s',
subnet_name, self._name
)
except ClientError as exc:
logger.error(
'An error occured while trying to update route table'
' for subnet %s: %s', subnet_name, exc
)
else:
logger.info(
'Associating subnet %s with route table %s.', subnet_name, self._name
)
try:
self._resource.associate_with_subnet(SubnetId=subnet.id)
except ClientError as exc:
logger.error(exc)
else:
logger.error(
'Unable to associate subnet %s with route table %s because it does'
' not exist. Operation not performed.', subnet_name, self._name
)
def _update_routes(self):
"""Manage routes for the current route table."""
# Routes that are currently set
current_routes = self._resource.custom_routes
# Routes defined in configuration
routes = []
# Routes that do not exist yes
new = []
# If a route is already defined for the given CIDR but has
# to be updated, we need to use a specific client method
modified = []
# Routes that are no longer defined in the configuration
trash = []
# Parse the configuration and populate route list
for item in self._ctx.get('routes') or []:
route = self._build_route(item)
if route:
routes.append(RouteWrapper(route))
# Search for new routes and routes that should be updated
for item in set(routes) - set(current_routes):
match = next(
(r for r in current_routes if r.destination == item.destination), None
)
if match:
modified.append(item)
else:
new.append(item)
# Search for routes that should be deleted
for item in set(current_routes) - set(routes):
match = next(
(r for r in routes if r.destination == item.destination), None
)
if not match:
trash.append(item)
for wrapper in modified:
logger.info('%s: replacing route - %s.', self._name, wrapper)
if all((wrapper.instance_id, wrapper.network_interface_id)):
# We must only pass the ``instance_id`` attribute in
# order to update a route with an EC2 instance target
del wrapper.network_interface_id
try:
self._client.replace_route(RouteTableId=self._resource.id, **wrapper.route)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
for wrapper in new:
logger.info('%s: creating route - %s.', self._name, wrapper)
if all((wrapper.instance_id, wrapper.network_interface_id)):
# We must only pass the ``instance_id`` attribute in
# order to create a route with an EC2 instance target
del wrapper.network_interface_id
try:
self._resource.create_route(**wrapper.route)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
for wrapper in trash:
logger.info('%s: removing route - %s.', self._name, wrapper)
try:
self._client.delete_route(
RouteTableId=self._resource.id,
DestinationCidrBlock=wrapper.destination
)
except (ClientError, ParamValidationError) as exc:
logger.error(exc)
def _build_route(self, item):
"""Load target objects for route definitions and return a route as a dictionary.
The following targets are supported:
- internet gateways
- virtual private gateways
- VPC peering connections
- EC2 instances
"""
filters = [{'Name': 'tag:Name', 'Values': [item['target_name']]}]
# internet gateway targets
if item['target_type'] == 'internet_gateway':
resp = self._client.describe_internet_gateways(Filters=filters)
gateways = resp.get('InternetGateways')
igw = gateways[0] if gateways else None
if igw:
return {
'GatewayId': igw['InternetGatewayId'],
'DestinationCidrBlock': item['destination']
}
# virtual private gateway targets
elif item['target_type'] == 'vpn_gateway':
resp = self._client.describe_vpn_gateways(Filters=filters)
gateways = resp.get('VpnGateways')
vgw = gateways[0] if gateways else None
if vgw:
return {
'GatewayId': vgw['VpnGatewayId'],
'DestinationCidrBlock': item['destination']
}
# VPC peering connection targets
elif item['target_type'] == 'vpc_peering_connection':
resp = self._client.describe_vpc_peering_connections(Filters=filters)
connections = resp.get('VpcPeeringConnections')
connection = connections[0] if connections else None
if connection:
return {
'VpcPeeringConnectionId': connection['VpcPeeringConnectionId'],
'DestinationCidrBlock': item['destination']
}
# EC2 instance targets
elif item['target_type'] == 'instance':
resp = self._client.describe_instances(Filters=filters)
reservations = resp.get('Reservations')
reservation = reservations[0] if reservations else {}
instances = reservation.get('Instances')
instance = instances[0] if instances else None
if instance:
route = {
'InstanceId': instance['InstanceId'],
'DestinationCidrBlock': item['destination']
}
interfaces = instance.get('NetworkInterfaces')
interface = interfaces[0] if interfaces else None
if interface:
route['NetworkInterfaceId'] = interface['NetworkInterfaceId']
return route
# Unsupported targets
else:
logger.error("%s: unsupported target type '%s'", self._name, item['target_type'])
return None
