def __init__(self): self.logger = get_logger(__name__) region_name = os.getenv('AWS_REGION', 'ap-northeast-1') # 環境変数 AWS_REGION が設定されていない場合は処理しない if region_name != '': session = Session(region_name=region_name) self.resource = session.resource('s3')
def gcp_resource(): sesh = Session(aws_access_key_id=conf.GCP_ACCESS_KEY, aws_secret_access_key=conf.GCP_SECRET_KEY) sesh.events.unregister( 'before-parameter-build.s3.ListObjects', set_list_objects_encoding_type_url) return sesh.resource('s3', endpoint_url=conf.GCP_ENDPOINT)
def test_create_subnet(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } subnet_ctx = { 'name': 'subnet01a', 'cidr_block': '10.0.10.0/25', 'zone': 'us-west-2a', 'vpc': 'vpc01', 'tags': { 'description': 'Test subnet (zone a) for VPC vpc01' } } tags = [ { 'Key': 'Name', 'Value': 'subnet01a' }, { 'Key': 'Description', 'Value': 'Test subnet (zone a) for VPC vpc01' } ] vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01a']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_subnet.SubnetWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Subnet' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=vpc_filters)) vpc = vpcs[0] # Create the subnet h = ec2_subnet.create_handler(subnet_ctx, self.credentials) h.create_resource() subnets = list(ec2.subnets.filter(Filters=subnet_filters)) subnet = subnets[0] self.assertEqual(len(subnets), 1) self.assertEqual(subnet.name, 'subnet01a') self.assertEqual(subnet.cidr_block, '10.0.10.0/25') self.assertEqual(subnet.availability_zone, 'us-west-2a') self.assertEqual(subnet.vpc_id, vpc.id) self.assertEqual(subnet.map_public_ip_on_launch, False) self.assertCountEqual(subnet.tags, tags)
def test_delete_internet_gateway(self): ctx = { 'name': 'igw01' } filters = [{'Name': 'tag:Name', 'Values': ['igw01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_igw.InternetGatewayWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.InternetGateway' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the internet gateway h = ec2_igw.create_handler(ctx, self.credentials) h.create_resource() gateways = list(ec2.internet_gateways.filter(Filters=filters)) self.assertEqual(len(gateways), 1) # We clear the resource cache to simulate a new # program execution with the 'delete' option base.BaseHandler._cache.clear() # Delete the internet gateway h.delete_resource() gateways = list(ec2.internet_gateways.filter(Filters=filters)) self.assertEqual(len(gateways), 0)
def to_parquet( df, bucket_name, prefix, retry_config, session_kwargs, client_kwargs, compression=None, flavor="spark", ): import pyarrow as pa from pyarrow import parquet as pq session = Session(**session_kwargs) client = session.resource("s3", **client_kwargs) bucket = client.Bucket(bucket_name) table = pa.Table.from_pandas(df) buf = pa.BufferOutputStream() pq.write_table(table, buf, compression=compression, flavor=flavor) response = retry_api_call( bucket.put_object, config=retry_config, Body=buf.getvalue().to_pybytes(), Key=prefix + str(uuid.uuid4()), ) return "s3://{0}/{1}".format(response.bucket_name, response.key)
def uploadFile(url): filename = url.split("/")[-1] fileextension = filename.split('.')[1] file = requests.get(url).content filepath = os.path.join(MEDIA_ROOT, filename) with open(filepath, 'wb') as destination: destination.write(file) file = open(filepath, 'rb') extension = FileUploadUtils.getContentType(fileextension) valid_file = True if extension is None: valid_file = False session = Session(aws_access_key_id=settings.AWS_ACCESS_KEY, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name=settings.AWS_REGION_NAME) s3 = session.resource('s3') file_key = FileUploadUtils.getFileKey() if valid_file: res = s3.Bucket(settings.AWS_BUCKET_NAME).put_object(Key=file_key, Body=file, ContentType=extension, ACL='public-read') data = {'key': file_key, 'file_name': filename, 'is_link': True} serializer = serializers.CreateFileUploadSerializer(data=data) if serializer.is_valid(): serializer.save() if os.path.isfile(filepath): os.remove(filepath) media = Media.objects.get(key=file_key) return media else: return None
def lambda_handler(event, context): # Resulted list ips = [] # Init Boto3 session sess = Session() asg_client = sess.client("autoscaling") ec2res = sess.resource("ec2") # Give me all the AutoScaling groups! for asg in asg_client.describe_auto_scaling_groups()['AutoScalingGroups']: # Give me all the instances in the AutoScaling group! for instance in asg['Instances']: # Is you healthy? if instance['HealthStatus'] == "Healthy": # Give me ze private address instance = ec2res.Instance(instance['InstanceId']) ips.append(instance.private_ip_address) return { "statusCode": 200, "body": json.dumps({ "ips": ips, }), }
def test_delete_vpc(self): ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_vpc.VpcWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Vpc' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=filters)) self.assertEqual(len(vpcs), 1) # We clear the resource cache to simulate a new # program execution with the 'delete' option base.BaseHandler._cache.clear() # Delete the VPC h.delete_resource() vpcs = list(ec2.vpcs.filter(Filters=filters)) self.assertEqual(len(vpcs), 0)
def generate_dummy_bucket(): session = Session(aws_access_key_id=DUMMY_ACCESS_KEY_ID, aws_secret_access_key=DUMMY_SECRET_ACCESS_KEY) s3 = session.resource('s3', region_name='us-east-1') # create a bucket s3.create_bucket(Bucket=DUMMY_BUCKET_NAME) # create some objects in this bucket # the root contain 7 objects: 4 files and 3 directories # the first entry in the tree should be the directory `cache`, since boto3 # returns the values sorted alphabetically. The last object should be the # file called `index.js`. files = ('index.js', 'avatar.jpg', '__init__.py', 'Makefile', 'css/app.less', 'css/base.css', 'js/vendor/angular.js', 'js/vendor/angular.min.js', 'js/vendor/latest/react.js', 'cache/foo.rb', 'cache/staticfiles/latest/index.js' 'cache/staticfiles/dummy.txt', 'cache/staticfiles/logs.txt') for file in files: content = string.ascii_letters * random.randint(1, 10) s3.meta.client.put_object(Bucket=DUMMY_BUCKET_NAME, Key=file, Body=content)
def to_parquet( df: "DataFrame", bucket_name: str, prefix: str, retry_config: RetryConfig, session_kwargs: Dict[str, Any], client_kwargs: Dict[str, Any], compression: str = None, flavor: str = "spark", ) -> str: import pyarrow as pa from pyarrow import parquet as pq session = Session(**session_kwargs) client = session.resource("s3", **client_kwargs) bucket = client.Bucket(bucket_name) table = pa.Table.from_pandas(df) buf = pa.BufferOutputStream() pq.write_table(table, buf, compression=compression, flavor=flavor) response = retry_api_call( bucket.put_object, config=retry_config, Body=buf.getvalue().to_pybytes(), Key=prefix + str(uuid.uuid4()), ) return f"s3://{response.bucket_name}/{response.key}"
def main(sg_name="asgard.ai", action="add", ip=None): """ """ # parse args assert action in ["add", "remove"] # get aws credential from env aws_region = os.environ.get("AWS_REGION", "eu-west-1") aws_access_key_id = os.environ["AWS_ACCESS_KEY_ID"] aws_secret_access_key = os.environ["AWS_SECRET_ACCESS_KEY"] # aws client session = Session( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region ) ec2 = session.resource('ec2') # my ip & security group my_ip = get_my_ip() + '/32' sg = get_security_group(ec2, name=sg_name) already_authorized_ip = list( set([x["CidrIp"] for y in sg.ip_permissions for x in y["IpRanges"]]) ) # add or remove my ip form security group if action == "add" and my_ip not in already_authorized_ip: add_ip_to_security_group(sg, my_ip) if ip is not None: add_ip_to_security_group(sg, ip + "/32") elif action == "remove" and my_ip in already_authorized_ip: remove_ip_to_security_group(sg, my_ip) if ip is not None: remove_ip_to_security_group(sg, ip + "/32")
def __init__(self, bucket_name, path=None, aws_access_key_id=None, aws_secret_access_key=None): # try to get the access key and secret key either from this object's # init, or from the global config. self._access_key = aws_access_key_id or config.aws_access_key_id self._secret_key = aws_secret_access_key or config.aws_secret_access_key if not (self._access_key and self._secret_key): raise ImproperlyConfiguredError # create a session using the credentials session = Session(aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key) # create an S3 resource self.s3 = session.resource(self.BOTO3_S3_RESOURCE_ID) self.client = self.s3.meta.client # ensure that the bucket exists, and then set the bucket name self.__ensure_bucket_exists(bucket_name) self.bucket_name = bucket_name # set the base path self.path = normalize_path(path) # set containers self.directories = [] self.files = [] # get the base tree and prepare the iterable self.__tree = self.__fetch_tree()
def copy_dynamo_schema(src_session: Session, dst_session: Session, tables: list = ()) -> None: """ Reads the DynamoDB in source region and creates the same tables in destination region. :param src_session: The aws session object of the source profile/region. :param dst_session: The aws session object of the destination profile/region. :param tables: A list of the tables to be copied. If empty all tables will be created. :return: None """ d_src_client = src_session.client('dynamodb') if not tables: tables = d_src_client.list_tables()['TableNames'] for name in tables: d_src_resource = src_session.resource('dynamodb') table = d_src_resource.Table(name) d_dest_resource = dst_session.resource('dynamodb') provisioned_throughput = table.provisioned_throughput if 'NumberOfDecreasesToday' in provisioned_throughput.keys(): del provisioned_throughput['NumberOfDecreasesToday'] params = { 'TableName': name, 'KeySchema': table.key_schema, 'AttributeDefinitions': table.attribute_definitions, 'ProvisionedThroughput': provisioned_throughput } if table.global_secondary_indexes: gsi = table.global_secondary_indexes gsi_copy = copy.deepcopy(gsi) for item in gsi: for key in item.keys(): if key not in ["IndexName", "KeySchema", "Projection", "ProvisionedThroughput"]: del gsi_copy[gsi.index(item)][key] if 'NumberOfDecreasesToday' in gsi_copy[gsi.index(item)]['ProvisionedThroughput'].keys(): del gsi_copy[gsi.index(item)]['ProvisionedThroughput']['NumberOfDecreasesToday'] params['GlobalSecondaryIndexes'] = gsi_copy d_dest_resource.create_table(**params) print(f'Created table {name}')
def __init__(self, config=None): Indexer.__init__(self, config=config) session = Session( aws_access_key_id=config['aws_access_key_id'], aws_secret_access_key=config['aws_secret_access_key'], region_name=config['aws_region']) ddb = session.resource('dynamodb') self.table = ddb.Table('nico_crawler')
def __init__(self, env_session: boto3.Session): self._s3 = env_session.client( 's3', config=botocore.client.Config( max_pool_connections=S3Service._MAX_POOL_SIZE)) self._s3_resource = env_session.resource('s3') self.__progress_total = 0 self.lock = Lock()
def __init__(self): self.logger = get_logger(__name__) region_name = os.getenv('AWS_REGION', 'ap-northeast-1') # 環境変数 AWS_REGION が設定されていない場合は処理しない if region_name != '': session = Session(region_name=region_name) self.resource = session.resource('dynamodb') self.default_table = 'bookbot-entry'
def test_create_dhcp_options(self): ctx = { 'name': 'dhcp01', 'domain_name': [ 'test01.us-west-2.aws' ], 'domain_name_servers': [ '10.0.10.2' ], 'tags': { 'description': 'DHCP options set for VPC vpc01' } } tags = [ { 'Key': 'Name', 'Value': 'dhcp01' }, { 'Key': 'Description', 'Value': 'DHCP options set for VPC vpc01' } ] dhcp_configurations = [ { 'Key': 'domain-name', 'Values': [{'Value': 'test01.us-west-2.aws'}] }, { 'Key': 'domain-name-servers', 'Values': [{'Value': '10.0.10.2'}] } ] filters = [{'Name': 'tag:Name', 'Values': ['dhcp01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_dhcp.DhcpOptionsWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.DhcpOptions' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the DHCP options set h = ec2_dhcp.create_handler(ctx, self.credentials) h.create_resource() dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters)) dhcp = dhcp_options_sets[0] self.assertEqual(len(dhcp_options_sets), 1) self.assertEqual(dhcp.name, 'dhcp01') self.assertCountEqual(dhcp.dhcp_configurations, dhcp_configurations) self.assertCountEqual(dhcp.tags, tags)
def read_dynamodb_data(table, recette, profile): """ Scan all item from dynamodb. :param table: String :return: Data in Dictionary Format. """ print('Connecting to AWS DynamoDb') session = Session(profile_name=profile) dynamodb_resource = session.resource('dynamodb') table = dynamodb_resource.Table(table) print('Downloading ', end='') keys = [] for item in table.attribute_definitions: keys.append(item['AttributeName']) keys_set = set(keys) item_count = table.item_count if recette is not None: raw_data = table.scan(FilterExpression=Attr("codeRecette").eq(recette)) else: raw_data = table.scan(FilterExpression=Attr("noEtape").eq("0")) if raw_data is None: return None items = raw_data['Items'] print('This is a ' + str(type(items))) items = strip_ssml_tags(items) fieldnames = set([]).union(get_keys(items)) cur_total = (len(items) + raw_data['Count']) if cur_total > item_count: percents = 99.99 else: percents = cur_total * 100 / item_count print("{} records ..... {:02.0f}%".format(raw_data['Count'], percents), end='\r') while raw_data.get('LastEvaluatedKey'): print('Downloading ', end='') raw_data = table.scan(ExclusiveStartKey=raw_data['LastEvaluatedKey']) items.extend(raw_data['Items']) fieldnames = fieldnames.union(get_keys(items)) cur_total = (len(items) + raw_data['Count']) if cur_total > item_count: percents = 99.99 else: percents = cur_total * 100 / item_count print("{} records ..... {:02.0f}%".format(raw_data['Count'], percents), end='\r') print() print("Total downloaded records: {}".format(len(items))) for fieldname in fieldnames: if fieldname not in keys_set: keys.append(fieldname) return {'items': items, 'keys': keys}
def test_delete_route_table_with_association(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } subnet_ctx = { 'name': 'subnet01a', 'cidr_block': '10.0.10.0/25', 'zone': 'us-west-2a', 'vpc': 'vpc01' } rt_ctx = { 'name': 'rt01', 'vpc': 'vpc01', 'subnets': [ 'subnet01a' ] } filters = [{'Name': 'tag:Name', 'Values': ['rt01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_rt.RouteTableWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.RouteTable' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the subnet h = ec2_subnet.create_handler(subnet_ctx, self.credentials) h.create_resource() # Create the route table h = ec2_rt.create_handler(rt_ctx, self.credentials) h.create_resource() route_tables = list(ec2.route_tables.filter(Filters=filters)) self.assertEqual(len(route_tables), 1) # We clear the resource cache to simulate a new # program execution with the 'delete' option base.BaseHandler._cache.clear() # Delete the route table h.delete_resource() route_tables = list(ec2.route_tables.filter(Filters=filters)) self.assertEqual(len(route_tables), 0)
def __ec2_detach_eks(session: Session, kubecontext: str, namespace: str, label_name: str) -> str: """ Utility method to detach instance from a given namespace. This returns a node IP, gets instance ID from the same and returns back the same for chaos operations :param session: :param kubecontext: :param namespace: :param label_name: :return: """ logger.info("Getting list of pods for namespace " + namespace) v1 = K8sUtils.init_k8s_client(kubecontext) ip_address_list = [] v1podlist = v1.list_namespaced_pod(namespace, label_selector=label_name).items if not v1podlist: raise ChaosTestException( "No pods are there in the namespace which was provided " + namespace) if len(v1podlist) == 0: raise ChaosTestException("List of pods is empty for namespace " + namespace) for v1pod in v1podlist: if v1pod.status.phase == "Running": ip_address_list.append(v1pod.status.host_ip) if len(ip_address_list) == 0: raise ChaosTestException( "nodes are not present matching pod label " + label_name) ip_address_list = list(dict.fromkeys(ip_address_list)) # list_node_ip_addresses = [] # http_nodes = v1.list_node(pretty='true') # # for i in range(len(http_nodes.items) - 1): # address_list = http_nodes.items[i].status.addresses # for address in address_list: # if address.type == 'InternalIP': # list_node_ip_addresses.append(address.address) ip_selected = random.choice(ip_address_list) ec2_resource = session.resource('ec2') response = ec2_resource.instances.filter(Filters=[{ 'Name': 'instance-state-name', 'Values': ['running'] }]) instance_id = '' for instance in response: if ip_selected == instance.private_ip_address: logger.info("Instance Id returned " + instance.id) instance_id = instance.id break if not instance_id: raise Exception("Not able to get instance id, exiting") return instance_id
def test_function(event, context): get_google_response() aws_session = Session(region_name=REGION) sqs = aws_session.resource('sqs') sqs_queue = sqs.get_queue_by_name(QueueName=QUEUE) print('SQS url:' + sqs_queue.url) sqs_queue.send_message(MessageBody='hey Jude') return 'success'
def import_dynamodb_items_to_es(table_name, aws_secret, aws_access, aws_region, event_source_arn, lambda_f, scan_limit): global reports global partSize global object_amount logger = logging.getLogger() logger.setLevel(logging.INFO) session = Session(aws_access_key_id=aws_access, aws_secret_access_key=aws_secret, region_name=aws_region) dynamodb = session.resource('dynamodb') logger.info('dynamodb: %s', dynamodb) ddb_table_name = table_name table = dynamodb.Table(ddb_table_name) logger.info('table: %s', table) ddb_keys_name = [a['AttributeName'] for a in table.attribute_definitions] logger.info('ddb_keys_name: %s', ddb_keys_name) response = None while True: if not response: response = table.scan(Limit=scan_limit) else: response = table.scan( ExclusiveStartKey=response['LastEvaluatedKey'], Limit=scan_limit) for i in response["Items"]: ddb_keys = {k: i[k] for k in i if k in ddb_keys_name} ddb_data = boto3.dynamodb.types.TypeSerializer().serialize(i)["M"] ddb_keys = boto3.dynamodb.types.TypeSerializer().serialize( ddb_keys)["M"] record = { "dynamodb": { "SequenceNumber": "0000", "Keys": ddb_keys, "NewImage": ddb_data }, "awsRegion": aws_region, "eventName": "INSERT", "eventSourceARN": event_source_arn, "eventSource": "aws:dynamodb" } partSize += 1 object_amount += 1 logger.info(object_amount) reports.append(record) if partSize >= 100: send_to_eslambda(reports, lambda_f) if 'LastEvaluatedKey' not in response: break if partSize > 0: send_to_eslambda(reports, lambda_f)
def boto_resource_dynamodb( session: _boto3.Session, opts: _intf.Options, ) -> _tp.Any: kw = opts.mapped_update({ "region": "region_name", "dynamodb_endpoint": "endpoint_url", }) return session.resource("dynamodb", **kw)
def test_delete_attached_internet_gateway(self): igw_ctx = { 'name': 'igw01' } vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'internet_gateway': 'igw01' } igw_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}] vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_igw.InternetGatewayWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.InternetGateway' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the internet gateway h = ec2_igw.create_handler(igw_ctx, self.credentials) h.create_resource() gateways = list(ec2.internet_gateways.filter(Filters=igw_filters)) igw = gateways[0] self.assertCountEqual(igw.attachments, []) # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=vpc_filters)) vpc = vpcs[0] # Test that the internet gateway has been attached igw.reload() attachments = [{'VpcId': vpc.id, 'State': 'available'}] self.assertCountEqual(igw.attachments, attachments) # We clear the resource cache to simulate a new # program execution with the 'delete' option base.BaseHandler._cache.clear() # Delete the internet gateway h = ec2_igw.create_handler(igw_ctx, self.credentials) h.delete_resource() gateways = list(ec2.internet_gateways.filter(Filters=igw_filters)) # The gateway was not deleted self.assertEqual(len(gateways), 1)
class Resource_Ec2 : def __init__(self,iamProfilename) : from boto3 import Session self.session = Session(profile_name=iamProfilename) self.ec2 = self.session.resource('ec2') #Get EC2 instance list def _getEntireListofCurrentInstances(self) : response = [{'instance.id':instance.id,'instance.state':instance.state} for instance in self.ec2.instances.all()] return(response) #Create and Run EC2 instance def _createNewInstances(self,imageId,minCount,maxCount,instanceType,keyName,securityGroupIds,iamInstanceProfile,instanceTags,userData) : instance = self.ec2.create_instances( #create_instances Arguments References : http://docs.aws.amazon.com/ko_kr/cli/latest/userguide/generate-cli-skeleton.html ImageId = imageId, MinCount = minCount, MaxCount = maxCount, InstanceType = instanceType, KeyName = keyName, SecurityGroupIds = securityGroupIds, IamInstanceProfile = iamInstanceProfile, TagSpecifications = [ { 'ResourceType': 'instance', #ResourceType SHOULD BE in TagSpecifications 'Tags': [ instanceTags, ] } ], UserData = userData, ) return(instance) #Stop EC2 instances def _stopInstances(self,instanceIds) : instance = self.ec2.instances.filter(InstanceIds=instanceIds) response = instance.stop() return(response) #Start EC2 instances def _startInstances(self,instanceIds) : instance = self.ec2.instances.filter(InstanceIds=instanceIds) #Case#01 instance type is not supported on specific AWS region(e.g. ap-northeast-2) : botocore.exceptions.ClientError: An error occurred (Unsupported) when calling the StartInstances operation: The requested configuration is currently not supported. Please check the documentation for supported configurations. #Case#01 Trouble Shooting : chek the limits and change the instance type available, https://docs.aws.amazon.com/ko_kr/AWSEC2/latest/UserGuide/ec2-resource-limits.html #Case#01 Ref : m5.large is not supported on ap-northeaset-2, https://aws.amazon.com/ko/blogs/korea/m5-the-next-generation-of-general-purpose-ec2-instances response = instance.start() return(response) #Terminate EC2 instances def _terminateInstances(self,instanceIds) : instance = self.ec2.instances.filter(InstanceIds=instanceIds) response = instance.terminate() return(response)
def aws_connectors(): session = Session( region_name=os.environ.get('AWS_DEFAULT_REGION'), aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID'), aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY')) sqs = session.resource('sqs') sqs_client = session.client('sqs') return sqs, sqs_client
def import_xls(request): form = Upload_XLForm(request.POST or None, request.FILES or None) if form.is_valid(): form.save() form = Upload_XLForm() obj = Upload_XL.objects.get(activated=False) s3_session = Session(aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY) bucket_object = s3_session.resource('s3').Bucket( AWS_STORAGE_BUCKET_NAME).Object(str(obj.file_name)) content = bucket_object.get()['Body'].read() workbook = xlrd.open_workbook_xls(file_contents=content) # book = xlrd.open_workbook(obj.file_name.url) sh = workbook.sheet_by_index(0) p_row = 1 while p_row < sh.nrows: # print(sh.row(p_row).value) user, created = User.objects.get_or_create( username=sh.row(p_row)[3].value, email=sh.row(p_row)[4].value, ) current_post = Post.objects.create( title=sh.row(p_row)[0].value, content=sh.row(p_row)[1].value, date_posted=datetime.datetime.strptime( sh.row(p_row)[2].value, "%Y-%m-%d %H:%M:%S"), author=user, likes=sh.row(p_row)[5].value, ) c_row = p_row + 3 try: while sh.row(c_row)[1].value != '': comment_user, c_created = User.objects.get_or_create( username=sh.row(c_row)[3].value, email=sh.row(c_row)[4].value, ) Comment.objects.create( post=current_post, author=comment_user, text=sh.row(c_row)[1].value, created_date=datetime.datetime.strptime( sh.row(c_row)[2].value, "%Y-%m-%d %H:%M:%S"), ) c_row += 1 except: pass p_row = c_row + 2 obj.activated = True obj.save() obj.delete() return render(request, 'blog/uploadXL.html', {'form': form})
def test_create_route_table(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } rt_ctx = { 'name': 'rt01', 'vpc': 'vpc01', 'tags': { 'description': 'Replace the default route table for VPC vpc01' } } tags = [ { 'Key': 'Name', 'Value': 'rt01' }, { 'Key': 'Description', 'Value': 'Replace the default route table for VPC vpc01' } ] vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] rt_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_rt.RouteTableWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.RouteTable' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=vpc_filters)) vpc = vpcs[0] # Create the route table h = ec2_rt.create_handler(rt_ctx, self.credentials) h.create_resource() route_tables = list(ec2.route_tables.filter(Filters=rt_filters)) rt = route_tables[0] self.assertEqual(len(route_tables), 1) self.assertEqual(rt.name, 'rt01') self.assertEqual(rt.vpc_id, vpc.id) self.assertCountEqual(rt.tags, tags)
def test_create_alert(self): session = Session() # Get the service resource dynamodb = session.resource('dynamodb') dynamodb.create_table( TableName='sso-dashboard-alert1', KeySchema=[ { 'AttributeName': 'alert_id', 'KeyType': 'HASH' } ], AttributeDefinitions=[ { 'AttributeName': 'alert_id', 'AttributeType': 'S' } ], ProvisionedThroughput={ 'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5 } ) self.table = dynamodb.Table('sso-dashboard-alert1') self.table.meta.client.get_waiter('table_exists').wait(TableName='sso-dashboard-alert1') a = alert.Alert() a.dynamodb = self.table alert_dict = { 'user_id': 'bob|ad|123456', 'message': 'foo', 'severity': 'HIGH' } # res = a.create(alert_dict=alert_dict) sample_alert_id = res['Attributes']['alert_id'] assert res['ResponseMetadata']['HTTPStatusCode'] == 200 res = a.find('bob|ad|123456') assert len(res) is 1 updated_alert_dict = { 'user_id': 'bob|ad|123456', 'message': 'foo', 'severity': 'MEDIUM' } res = a.update(alert_id=sample_alert_id, alert_dict=updated_alert_dict) assert res['ResponseMetadata']['HTTPStatusCode'] == 200 res = a.destroy(alert_id=sample_alert_id, user_id='bob|ad|123456') assert res['ResponseMetadata']['HTTPStatusCode'] == 200
def ReadBucket(bucket_type, bucket_name, prefix=''): from boto3 import Session from google.cloud import storage try: blobs = [] # AWS S3 if bucket_type == 's3' or bucket_type == 'aws': #download_url = f"http://{bucket_name}.s3.amazonaws.com/" s3_client = Session() s3_resource = s3_client.resource('s3') s3_bucket = s3_resource.Bucket(bucket_name) for obj in s3_bucket.objects.filter(Prefix=prefix): #print(obj.__dict__) blobs.append(obj.key) # Google Cloud Storage if bucket_type == 'gs' or bucket_type == 'gcs': #download_url = f"http://{bucket_name}.storage.googleapis.com/" storage_client = storage.Client.create_anonymous_client() for blob in storage_client.list_blobs(bucket_name, prefix=prefix): #print(blob.__dict__) blobs.append(blob.name) files = [] dirs = {} for blob in blobs: #print(blob) file_name = blob.split('/')[-1] if '/' in blob: parent_dir = blob.split('/')[-2] else: parent_dir = "" path_match = prefix.split('/')[-1] print(file_name, parent_dir, path_match) if parent_dir == path_match: files.append(file_name) if '/' in blob: # Directory handling _ = blob.replace(prefix, '') if '/' in _: _ = _.split('/')[-2] print(_) if not _ in dirs: dirs[_] = True return dict(files=files, dirs=dirs) except Exception as e: raise (e)
def setup_resources(self): if self.aws_access_key_id and self.aws_secret_access_key: session = Session(aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, region_name=self.aws_region_name) self.resource = session.resource('dynamodb') else: self.resource = boto3.resource('dynamodb', aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, region_name=self.aws_region_name) self.table = self.resource.Table(self.table_name)
class AWSClient(object): """Manages automatically creating and destroying clients to AWS services.""" def __init__(self, resource, config, credentials=None, region_name=None): """Constructor :param resource: AWS specific token for resource type. e.g., 's3', 'sqs', etc. :type resource: string :param config: Resource specific configuration :type config: :class:`botocore.client.Config` :param credentials: Authentication values needed to access AWS. If no credentials are passed, then IAM role-based access is assumed. :type credentials: :class:`util.aws.AWSCredentials` :param region_name: The AWS region the resource resides in. :type region_name: string """ self.credentials = credentials self.region_name = region_name self._client = None self._resource_name = resource self._config = config def __enter__(self): """Callback handles creating a new client for AWS access.""" logger.debug('Setting up AWS client...') session_args = {} if self.credentials: session_args['aws_access_key_id'] = self.credentials.access_key_id session_args['aws_secret_access_key'] = self.credentials.secret_access_key if self.region_name: session_args['region_name'] = self.region_name self._session = Session(**session_args) self._client = self._session.client(self._resource_name, config=self._config) self._resource = self._session.resource(self._resource_name, config=self._config) return self def __exit__(self, type, value, traceback): """Callback handles destroying an existing client.""" pass @staticmethod def instantiate_credentials_from_config(config): if 'credentials' in config and config['credentials']: credentials_dict = config['credentials'] if 'access_key_id' not in credentials_dict or not credentials_dict['access_key_id']: raise InvalidAWSCredentials('"credentials" requires "access_key_id" to be populated') if 'secret_access_key' not in credentials_dict or not credentials_dict['secret_access_key']: raise InvalidAWSCredentials('"credentials" requires "secret_access_key" to be populated') return AWSCredentials(credentials_dict['access_key_id'], credentials_dict['secret_access_key'])
def test_create_security_group(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } sg_ctx = { 'name': 'sg01a', 'description': 'Test security group sg01a', 'vpc': 'vpc01' } tags = [ { 'Key': 'Name', 'Value': 'sg01a' } ] vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] sg_filters = [{'Name': 'tag:Name', 'Values': ['sg01a']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_sg.SecurityGroupWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.SecurityGroup' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=vpc_filters)) vpc = vpcs[0] # Create the security group h = ec2_sg.create_handler(sg_ctx, self.credentials) h.create_resource() security_groups = list(ec2.security_groups.filter(Filters=sg_filters)) sg = security_groups[0] self.assertEqual(len(security_groups), 1) self.assertEqual(sg.name, 'sg01a') # Security groups have a dedicated attribute for their name self.assertEqual(sg.name, sg.group_name) self.assertEqual(sg.vpc_id, vpc.id) self.assertCountEqual(sg.tags, tags)
def get_table( table_name: str, profile: Optional[str] = None, region: Optional[str] = None, retries: Optional[int] = None, ): from boto3 import Session from botocore.config import Config session = Session(profile_name=profile, region_name=region) config = Config(retries={"max_attempts": retries}) if retries else Config() dynamodb = session.resource("dynamodb", config=config) return dynamodb.Table(table_name)
def __init__(self, table_name, aws_access_key_id=None, aws_secret_access_key=None, aws_region_name='us-west-2'): if aws_access_key_id and aws_secret_access_key: session = Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, region_name=aws_region_name) self.resource = session.resource('dynamodb') else: self.resource = boto3.resource('dynamodb') self.table = self.resource.Table(table_name)
class Resource_Dynamodb : def __init__(self,iamProfilename,tableName) : from boto3 import Session self.session = Session(profile_name=iamProfilename) self.dynamodb = self.session.resource('dynamodb') self.table = self.dynamodb.Table(tableName) def _getCountOfQueryResult(self, partitionKey, condiTion): from boto3.dynamodb.conditions import Key # Reference1 : https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.Python.04.html # Reference2 : http://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.Client.query response = self.table.query( KeyConditionExpression = Key(partitionKey).eq(condiTion), Select = 'COUNT' ) return(response)
def upload_file_by_file(file): milli_sec = str(datetime.datetime.now()) filename = str(milli_sec) + '.pdf' print(file) session = Session(aws_access_key_id=settings.AWS_ACCESS_KEY, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name=settings.AWS_REGION_NAME) s3 = session.resource('s3') file_key = FileUploadUtils.getFileKey() res = s3.Bucket(settings.AWS_BUCKET_NAME).put_object(Key=file_key, Body=file, ContentType='application/pdf', ACL='public-read') data = {'key': file_key, 'file_name': filename, 'is_link': False} serializer = serializers.CreateFileUploadSerializer(data=data) if serializer.is_valid(): serializer.save() media = Media.objects.get(key=file_key) return media
class Accessor(): def __init__(self, bucket_name, profile_name=None): """ s3との接続を確立する """ self.profile_name = profile_name self.bucket_name = bucket_name self.session = Session(profile_name=profile_name) self.s3 = self.session.resource('s3') self.bucket = self.s3.Bucket(bucket_name) def upload_file(self, src, dst): """ src:アップロードするファイルのパス dst:アップロード先(ファイル名含む) 例:src=/tmp/hoge.tsv, dst=HOGE/FUGA/hoge.tsv 設定した bucketの HOGE/FUGAにhoge.tsv という名前でアップロードする。ディレクトリは自動で作成される """ self.bucket.upload_file(src, dst) def download_file(self, src, dst): """ src:ダウンロードするファイルのパス dst:ダウンロード先(ファイル名含む) 例:src=HOGE/FUGA/hoge.tsv, dst=/hoge/fuga.tsv 設定した bucketの HOGE/FUGAにhoge.tsv を /hoge/fuga.tsv という名前でダウンロードする """ if not os.path.exists(os.path.dirname(dst)): os.makedirs(os.path.dirname(dst)) self.bucket.download_file(src, dst) def download_directory(self, src_dir, dst_dir): """ 指定したディレクトリ内のファイルをダウンロードする src_dir:ダウンロードするディレクトリ dst_dir:ダウンロード先ディレクトリ 例:src=HOGE/FUGA, dst=/hoge 設定した bucketの HOGE/FUGA 内のファイルを を /hoge 以下にダウンロードする """ total = len(list(self.bucket.objects.filter(Prefix=src_dir))) for obj in tqdm(self.bucket.objects.filter(Prefix=src_dir), total=total): logname = os.path.basename(obj.key) dst = os.path.join(dst_dir, logname) self.download_file(obj.key, dst)
def test_delete_security_group(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } sg_ctx = { 'name': 'sg01a', 'description': 'Test security group sg01a', 'vpc': 'vpc01' } filters = [{'Name': 'tag:Name', 'Values': ['sg01a']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_sg.SecurityGroupWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.SecurityGroup' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the security group h = ec2_sg.create_handler(sg_ctx, self.credentials) h.create_resource() security_groups = list(ec2.security_groups.filter(Filters=filters)) self.assertEqual(len(security_groups), 1) # We clear the resource cache to simulate a new # program execution with the 'delete' option base.BaseHandler._cache.clear() # Delete the security group h.delete_resource() security_groups = list(ec2.security_groups.filter(Filters=filters)) self.assertEqual(len(security_groups), 0)
def test_attach_internet_gateway(self): igw_ctx = { 'name': 'igw01' } vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'internet_gateway': 'igw01' } igw_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}] vpc_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_vpc.VpcWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Vpc' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the internet gateway h = ec2_igw.create_handler(igw_ctx, self.credentials) h.create_resource() gateways = list(ec2.internet_gateways.filter(Filters=igw_filters)) igw = gateways[0] self.assertCountEqual(igw.attachments, []) # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=vpc_filters)) vpc = vpcs[0] # Test that the internet gateway has been attached igw.reload() attachments = [{'VpcId': vpc.id, 'State': 'available'}] self.assertCountEqual(igw.attachments, attachments)
def deleteFile(key): media = Media.objects.get(id=key) session = Session(aws_access_key_id=settings.AWS_ACCESS_KEY, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY, region_name=settings.AWS_REGION_NAME) s3 = session.resource('s3') my_bucket = s3.Bucket(settings.AWS_BUCKET_NAME) response = my_bucket.delete_objects( Delete={ 'Objects': [ { 'Key': media.key } ] } ) media.delete() return response
def test_create_vpc(self): ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'tags': { 'description': 'VPC vpc01 (subnet01a & subnet01b)' } } tags = [ { 'Key': 'Name', 'Value': 'vpc01' }, { 'Key': 'Description', 'Value': 'VPC vpc01 (subnet01a & subnet01b)' } ] filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_vpc.VpcWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Vpc' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(ctx, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=filters)) vpc = vpcs[0] self.assertEqual(len(vpcs), 1) self.assertEqual(vpc.name, 'vpc01') self.assertEqual(vpc.cidr_block, '10.0.10.0/24') self.assertCountEqual(vpc.tags, tags)
def test_create_internet_gateway(self): ctx = { 'name': 'igw01', 'tags': { 'description': 'Internet gateway for VPC vpc01' } } tags = [ { 'Key': 'Name', 'Value': 'igw01' }, { 'Key': 'Description', 'Value': 'Internet gateway for VPC vpc01' } ] filters = [{'Name': 'tag:Name', 'Values': ['igw01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_igw.InternetGatewayWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.InternetGateway' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the internet gateway h = ec2_igw.create_handler(ctx, self.credentials) h.create_resource() gateways = list(ec2.internet_gateways.filter(Filters=filters)) igw = gateways[0] self.assertEqual(len(gateways), 1) self.assertEqual(igw.name, 'igw01') self.assertCountEqual(igw.tags, tags) self.assertCountEqual(igw.attachments, [])
def test_delete_dhcp_options_set(self): ctx = { 'name': 'dhcp01', 'domain_name': [ 'test01.us-west-2.aws' ], 'domain_name_servers': [ '10.0.10.2' ] } filters = [{'Name': 'tag:Name', 'Values': ['dhcp01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_dhcp.DhcpOptionsWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.DhcpOptions' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the DHCP options set h = ec2_dhcp.create_handler(ctx, self.credentials) h.create_resource() dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters)) self.assertEqual(len(dhcp_options_sets), 1) # We clear the resource cache to simulate a new # program execution with the 'delete' option base.BaseHandler._cache.clear() # Delete the DHCP options set h.delete_resource() dhcp_options_sets = list(ec2.dhcp_options_sets.filter(Filters=filters)) self.assertEqual(len(dhcp_options_sets), 0)
class Website: def __init__(self, bucket_name, file_location, access_key, secret_access_key, region): self.bucket_name = bucket_name self.file_location = file_location self.session = Session( aws_access_key_id=access_key, aws_secret_access_key=secret_access_key, region_name=region ) self.s3 = self.session.resource("s3") self.bucket = self.s3.Bucket(bucket_name) def upload_files_to_amazon(self): for root, directory, files in os.walk(self.file_location): for f in files: file_location = os.path.abspath(os.path.join(root, f)) file_key = file_location.replace(self.file_location, "") # Make sure the documents are correctly presented when they get on the server. extra_args = dict() if f.endswith(".css"): extra_args["ContentType"] = "text/css" elif f.endswith(".json"): extra_args["ContentType"] = "application/json" elif f.endswith(".js"): extra_args["ContentType"] = "application/javascript" elif f.endswith(".html"): extra_args["ContentType"] = "text/html" elif f.endswith(".txt"): extra_args["ContentType"] = "text/plain" elif f.endswith(".pdf"): extra_args["ContentType"] = "application/pdf" elif f.endswith(".jpeg"): extra_args["ContentType"] = "image/jpeg" elif f.endswith(".png"): extra_args["ContentType"] = "image/png" elif f.endswith(".bmp"): extra_args["ContentType"] = "image/bmp" elif f.endswith(".gif"): extra_args["ContentType"] = "image/gif" self.s3.meta.client.upload_file(file_location, self.bucket_name, file_key, extra_args)
def test_revoke_egress_rule(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } sg_ctx1 = { 'name': 'sg01a', 'description': 'Test security group sg01a', 'vpc': 'vpc01', 'egress': [ { 'ip_protocol': '-1', 'from_port': -1, 'to_port': -1, 'destinations': [ '0.0.0.0/0' ] } ] } sg_ctx2 = { 'name': 'sg01a', 'description': 'Test security group sg01a', 'vpc': 'vpc01' } ip_permission = { 'IpProtocol': '-1', 'FromPort': -1, 'ToPort': -1, 'IpRanges': [ { 'CidrIp': '0.0.0.0/0' } ], 'UserIdGroupPairs': [] } rule = ec2_sg.RuleWrapper(ip_permission, flow='destination') filters = [{'Name': 'tag:Name', 'Values': ['sg01a']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_sg.SecurityGroupWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.SecurityGroup' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the security group h = ec2_sg.create_handler(sg_ctx1, self.credentials) h.create_resource() security_groups = list(ec2.security_groups.filter(Filters=filters)) sg = security_groups[0] self.assertEqual(len(security_groups), 1) self.assertEqual(len(sg.egress_rules), 1) self.assertIn(rule, sg.egress_rules) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the security group h = ec2_sg.create_handler(sg_ctx2, self.credentials) h.update_resource() security_groups = list(ec2.security_groups.filter(Filters=filters)) sg = security_groups[0] self.assertEqual(len(security_groups), 1) self.assertEqual(len(sg.egress_rules), 0)
class InternetGatewayHandler(base.BaseHandler): """Manage ``EC2.InternetGateway`` object lifecycle.""" def __init__(self, ctx, credentials): """Create boto3 session and clients.""" super().__init__() # Create a boto session and add extra features # to the base class EC2.InternetGateway event = "creating-resource-class.ec2.InternetGateway" self._session = Session(**credentials) self._session.events.register(event, _add_wrapper) # boto3 clients self._ec2 = self._session.resource("ec2") self._client = self._session.client("ec2") # Context for the resource we want # to create, update or delete self._ctx = ctx self._name = ctx.get("name") def create_resource(self): """Create an internet gateway, or update it if it already exists.""" self._resource = self._load(self._name, manager_name="internet_gateways") if self._resource: logger.info("Internet gateway %s already exists.", self._name) else: self._create_internet_gateway() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource def update_resource(self): """Update an internet gateway object.""" self._resource = self._load(self._name, manager_name="internet_gateways") if self._resource: self._update_internet_gateway() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource else: logger.error("Internet gateway %s does not exist.", self._name) def delete_resource(self): """Delete an ``InternetGateway`` object.""" self._resource = self._load(self._name, manager_name="internet_gateways") if self._resource: logger.info("Removing internet gateway %s.", self._name) if self._resource.attachments: # If the internet gateway is attached to a VPC, # we don't take the responsibility to detach it vpc_id = self._resource.attachments[0]["VpcId"] logger.warning( "The internet gateway %s is currently attached to VPC %s." " Operation not performed.", self._name, vpc_id, ) else: try: self._client.delete_internet_gateway(InternetGatewayId=self._resource.id) except ClientError as exc: logger.error(exc) else: logger.error("Internet gateway %s does not exist.", self._name) def _create_internet_gateway(self): """Create an ``InternetGateway`` object.""" logger.info("Creating internet gateway %s.", self._name) try: self._resource = self._ec2.create_internet_gateway() except ClientError as exc: logger.error(exc) return # Set the value of the 'Name' tag self._resource.name = self._name # Update other object attributes self._update_internet_gateway() def _update_internet_gateway(self): """Update a ``InternetGateway`` object.""" self._update_tags()
def test_update_security_group(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } sg_ctx1 = { 'name': 'sg01b', 'description': 'Test security group sg01b', 'vpc': 'vpc01', 'tags': { 'stack': 'Test', 'owner': 'Team A' } } sg_ctx2 = { 'name': 'sg01b', 'description': 'Test security group sg01b', 'vpc': 'vpc01', 'tags': { 'stack': 'Production', 'platform': 'app01' } } tags1 = [ { 'Key': 'Name', 'Value': 'sg01b' }, { 'Key': 'Stack', 'Value': 'Test' }, { 'Key': 'Owner', 'Value': 'Team A' } ] tags2 = [ { 'Key': 'Name', 'Value': 'sg01b' }, { 'Key': 'Stack', 'Value': 'Production' }, { 'Key': 'Platform', 'Value': 'app01' } ] sg_filters = [{'Name': 'tag:Name', 'Values': ['sg01b']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_sg.SecurityGroupWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.SecurityGroup' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the security group h = ec2_sg.create_handler(sg_ctx1, self.credentials) h.create_resource() security_groups = list(ec2.security_groups.filter(Filters=sg_filters)) sg = security_groups[0] sg_id = sg.id self.assertEqual(len(security_groups), 1) self.assertCountEqual(sg.tags, tags1) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the security group h = ec2_sg.create_handler(sg_ctx2, self.credentials) h.update_resource() security_groups = list(ec2.security_groups.filter(Filters=sg_filters)) sg = security_groups[0] self.assertEqual(len(security_groups), 1) self.assertEqual(sg.id, sg_id) self.assertCountEqual(sg.tags, tags2)
def test_update_subnet(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } subnet_ctx1 = { 'name': 'subnet01b', 'cidr_block': '10.0.10.128/25', 'zone': 'us-west-2b', 'vpc': 'vpc01', 'tags': { 'description': 'Test subnet (zone b) for VPC vpc01', 'stack': 'Test', 'owner': 'Team A' } } subnet_ctx2 = { 'name': 'subnet01b', 'cidr_block': '10.0.10.128/25', 'zone': 'us-west-2b', 'vpc': 'vpc01', 'tags': { 'description': 'Test subnet (zone b) for VPC vpc01', 'stack': 'Production' } } tags1 = [ { 'Key': 'Name', 'Value': 'subnet01b' }, { 'Key': 'Description', 'Value': 'Test subnet (zone b) for VPC vpc01' }, { 'Key': 'Stack', 'Value': 'Test' }, { 'Key': 'Owner', 'Value': 'Team A' } ] tags2 = [ { 'Key': 'Name', 'Value': 'subnet01b' }, { 'Key': 'Description', 'Value': 'Test subnet (zone b) for VPC vpc01' }, { 'Key': 'Stack', 'Value': 'Production' } ] subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01b']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_subnet.SubnetWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Subnet' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the subnet h = ec2_subnet.create_handler(subnet_ctx1, self.credentials) h.create_resource() subnets = list(ec2.subnets.filter(Filters=subnet_filters)) subnet = subnets[0] subnet_id = subnet.id self.assertEqual(len(subnets), 1) self.assertCountEqual(subnet.tags, tags1) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the subnet h = ec2_subnet.create_handler(subnet_ctx2, self.credentials) h.update_resource() subnets = list(ec2.subnets.filter(Filters=subnet_filters)) subnet = subnets[0] self.assertEqual(len(subnets), 1) self.assertEqual(subnet.id, subnet_id) self.assertCountEqual(subnet.tags, tags2)
from __future__ import print_function from HTMLParser import HTMLParser import requests from boto3 import Session from boto3.dynamodb.conditions import Key, Attr import uuid,time session = Session(region_name='eu-west-1') dynamodb = session.resource('dynamodb') table = dynamodb.Table('GameMetrics') def lambda_handler(event, context): class reviews(HTMLParser): def __init__(self): HTMLParser.__init__(self) self.inLink = False self.dataArray = [] self.countLanguages = 0 self.lasttag = None self.lastname = None self.lastvalue = None def handle_starttag(self, tag, attrs): self.inLink = False if tag == 'span': for name, value in attrs: if name == 'class' and value =='user_reviews_count': self.countLanguages += 1
def test_update_map_public_ip_on_launch(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } subnet_ctx1 = { 'name': 'subnet01b', 'cidr_block': '10.0.10.128/25', 'zone': 'us-west-2b', 'vpc': 'vpc01' } subnet_ctx2 = { 'name': 'subnet01b', 'cidr_block': '10.0.10.128/25', 'zone': 'us-west-2b', 'vpc': 'vpc01', 'map_public_ip_on_launch': True } subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01b']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_subnet.SubnetWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Subnet' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the subnet h = ec2_subnet.create_handler(subnet_ctx1, self.credentials) h.create_resource() subnets = list(ec2.subnets.filter(Filters=subnet_filters)) subnet = subnets[0] subnet_id = subnet.id self.assertEqual(len(subnets), 1) self.assertEqual(subnet.name, 'subnet01b') self.assertEqual(subnet.cidr_block, '10.0.10.128/25') self.assertEqual(subnet.availability_zone, 'us-west-2b') self.assertEqual(subnet.map_public_ip_on_launch, False) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the subnet h = ec2_subnet.create_handler(subnet_ctx2, self.credentials) h.update_resource() subnets = list(ec2.subnets.filter(Filters=subnet_filters)) subnet = subnets[0] self.assertEqual(len(subnets), 1) self.assertEqual(subnet.id, subnet_id) self.assertEqual(subnet.map_public_ip_on_launch, True)
def test_delete_route(self): igw_ctx = { 'name': 'igw01' } vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'internet_gateway': 'igw01' } rt_ctx1 = { 'name': 'rt01', 'vpc': 'vpc01', 'routes': [ { 'destination': '0.0.0.0/0', 'target_name': 'igw01', 'target_type': 'internet_gateway' } ] } rt_ctx2 = { 'name': 'rt01', 'vpc': 'vpc01', 'routes': None } igw_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}] rt_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_rt.RouteTableWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.RouteTable' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the internet gateway h = ec2_igw.create_handler(igw_ctx, self.credentials) h.create_resource() internet_gateways = list(ec2.internet_gateways.filter(Filters=igw_filters)) igw = internet_gateways[0] # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the route table h = ec2_rt.create_handler(rt_ctx1, self.credentials) h.create_resource() route_tables = list(ec2.route_tables.filter(Filters=rt_filters)) rt = route_tables[0] route = { 'GatewayId': igw.id, 'DestinationCidrBlock': '0.0.0.0/0', 'Origin': 'CreateRoute', 'State': 'active' } route_wrapper = ec2_rt.RouteWrapper(route) self.assertEqual(len(rt.routes), 2) # the new route + the default VPC route self.assertIn(route_wrapper, rt.custom_routes) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the route table h = ec2_rt.create_handler(rt_ctx2, self.credentials) h.update_resource() route_tables = list(ec2.route_tables.filter(Filters=rt_filters)) rt = route_tables[0] self.assertEqual(len(rt.routes), 1) # the default VPC route self.assertCountEqual(rt.custom_routes, [])
def test_update_route_table(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } rt_ctx1 = { 'name': 'rt01', 'vpc': 'vpc01', 'tags': { 'description': 'Replace the default route table for VPC vpc01', 'stack': 'Test', 'owner': 'Team A' } } rt_ctx2 = { 'name': 'rt01', 'vpc': 'vpc01', 'tags': { 'description': 'Replace the default route table for VPC vpc01', 'stack': 'Production' } } tags1 = [ { 'Key': 'Name', 'Value': 'rt01' }, { 'Key': 'Description', 'Value': 'Replace the default route table for VPC vpc01' }, { 'Key': 'Stack', 'Value': 'Test' }, { 'Key': 'Owner', 'Value': 'Team A' } ] tags2 = [ { 'Key': 'Name', 'Value': 'rt01' }, { 'Key': 'Description', 'Value': 'Replace the default route table for VPC vpc01' }, { 'Key': 'Stack', 'Value': 'Production' } ] rt_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_rt.RouteTableWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.RouteTable' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the route table h = ec2_rt.create_handler(rt_ctx1, self.credentials) h.create_resource() route_tables = list(ec2.route_tables.filter(Filters=rt_filters)) rt = route_tables[0] rt_id = rt.id self.assertEqual(len(route_tables), 1) self.assertCountEqual(rt.tags, tags1) # Update the route table h = ec2_rt.create_handler(rt_ctx2, self.credentials) h.update_resource() route_tables = list(ec2.route_tables.filter(Filters=rt_filters)) rt = route_tables[0] self.assertEqual(len(route_tables), 1) self.assertEqual(rt.id, rt_id) self.assertCountEqual(rt.tags, tags2)
class SecurityGroupHandler(BaseHandler): """Manage ``EC2.SecurityGroup`` object lifecycle.""" def __init__(self, ctx, credentials): """Create boto3 session and clients.""" super().__init__() # Create a boto session and add extra # features to the base class EC2.SecurityGroup event = 'creating-resource-class.ec2.SecurityGroup' self._session = Session(**credentials) self._session.events.register(event, _add_wrapper) # boto3 clients self._ec2 = self._session.resource('ec2') self._client = self._session.client('ec2') # Context for the resource we want # to create, update or delete self._ctx = ctx self._name = ctx.get('name') def create_resource(self): """Create a security group, or update it if it already exists.""" self._resource = self._load(self._name, manager_name='security_groups') if self._resource: logger.info('Security group %s already exists.', self._name) else: self._create_security_group() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource def update_resource(self): """Update a security group object.""" self._resource = self._load(self._name, manager_name='security_groups') if self._resource: self._update_security_group() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource else: logger.error('Security group %s does not exist.', self._name) @retry(**retry_params) def delete_resource(self): """Delete a ``SecurityGroup`` object.""" self._resource = self._load(self._name, manager_name='security_groups') if self._resource: logger.info('Removing security group %s.', self._name) try: self._resource.delete(GroupId=self._resource.id) except ClientError as exc: # This is intended to prevent potential errors when trying # to remove security groups that may depend on each other. # If a 'DependencyViolation' is raised, we try to delete # the security group again after a short delay. if exc.response['Error']['Code'] == 'DependencyViolation': logger.info( 'Security group %s has a dependency and can ' 'not be removed. Trying again...', self._name ) raise DependencyViolationError else: logger.error(exc) else: logger.error('Security group %s does not exist.', self._name) def _create_security_group(self): """Create a ``SecurityGroup`` object.""" vpc_name = self._ctx['vpc'] vpc = self._load(vpc_name, manager_name='vpcs') if vpc: logger.info('Creating security group %s.', self._name) try: self._resource = self._ec2.create_security_group( GroupName=self._name, Description=self._ctx['description'], VpcId=vpc.id ) except (ClientError, ParamValidationError) as exc: logger.error(exc) return # Remove the default outbound rule authorizing all outgoing traffic ip_permission = { 'IpProtocol': '-1', 'FromPort': -1, 'ToPort': -1, 'IpRanges': [{'CidrIp': '0.0.0.0/0'}], 'UserIdGroupPairs': [] } logger.info('%s: revoking default outbound rule.', self._name) try: self._resource.revoke_egress(IpPermissions=[ip_permission]) except (ClientError, ParamValidationError) as exc: logger.error(exc) # Set the value of the 'Name' tag self._resource.name = self._name # Update other object attributes self._update_security_group() else: logger.error( 'Unable to create security group %s because VPC %s does not' ' exist. Operation not performed.', self._name, vpc_name ) def _update_security_group(self): """Update a ``SecurityGroup`` object.""" self._update_tags() self._update_ingress_rules() self._update_egress_rules() def _update_ingress_rules(self): """Manage security group ingress rules.""" current_ingress_rules = self._resource.ingress_rules new_ingress_rules = [] to_authorize = [] to_revoke = [] for rule in self._ctx.get('ingress', []): for source in rule['sources']: ip_permission = { 'IpProtocol': str(rule['ip_protocol']), 'FromPort': int(rule['from_port']), 'ToPort': int(rule['to_port']) } try: ipaddress.ip_network(source) # 'source' attribute is a valid CIDR ip_permission['IpRanges'] = [{'CidrIp': source}] ip_permission['UserIdGroupPairs'] = [] except ValueError: # Assuming we got a security group ID sg = self._load(source, manager_name='security_groups') if sg: pair = {'GroupId': sg.id, 'UserId': sg.owner_id} ip_permission['UserIdGroupPairs'] = [pair] ip_permission['IpRanges'] = [] else: logger.error( "%s: invalid value (source security group " "'%s' does not exist).", self._name, source ) continue new_ingress_rules.append( RuleWrapper(ip_permission, flow='source') ) for rule in set(new_ingress_rules) - set(current_ingress_rules): logger.info('%s: authorizing inbound rule %s.', self._name, rule) to_authorize.append(rule.ip_permission) for rule in set(current_ingress_rules) - set(new_ingress_rules): logger.info('%s: revoking inbound rule %s.', self._name, rule) to_revoke.append(rule.ip_permission) try: if to_authorize: self._resource.authorize_ingress(IpPermissions=to_authorize) if to_revoke: self._resource.revoke_ingress(IpPermissions=to_revoke) except (ClientError, ParamValidationError) as exc: logger.error(exc) def _update_egress_rules(self): """Manage security group egress rules.""" current_egress_rules = self._resource.egress_rules new_egress_rules = [] to_authorize = [] to_revoke = [] for rule in self._ctx.get('egress', []): for destination in rule['destinations']: ip_permission = { 'IpProtocol': str(rule['ip_protocol']), 'FromPort': int(rule['from_port']), 'ToPort': int(rule['to_port']) } try: ipaddress.ip_network(destination) # 'destination' attribute is a valid CIDR ip_permission['IpRanges'] = [{'CidrIp': destination}] ip_permission['UserIdGroupPairs'] = [] except ValueError: # Assuming we got a security group ID sg = self._load( destination, manager_name='security_groups' ) if sg: pair = {'GroupId': sg.id, 'UserId': sg.owner_id} ip_permission['UserIdGroupPairs'] = [pair] ip_permission['IpRanges'] = [] else: logger.error( "%s: invalid value (destination security group " "'%s' does not exist).", self._name, destination ) continue new_egress_rules.append( RuleWrapper(ip_permission, flow='destination') ) for rule in set(new_egress_rules) - set(current_egress_rules): logger.info('%s: authorizing outbound rule %s.', self._name, rule) to_authorize.append(rule.ip_permission) for rule in set(current_egress_rules) - set(new_egress_rules): logger.info('%s: revoking outbound rule %s.', self._name, rule) to_revoke.append(rule.ip_permission) try: if to_authorize: self._resource.authorize_egress(IpPermissions=to_authorize) if to_revoke: self._resource.revoke_egress(IpPermissions=to_revoke) except (ClientError, ParamValidationError) as exc: logger.error(exc)
def test_change_subnet_association(self): vpc_ctx = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24' } subnet_ctx = { 'name': 'subnet01a', 'cidr_block': '10.0.10.0/25', 'zone': 'us-west-2a', 'vpc': 'vpc01' } rt_ctx1 = { 'name': 'rt01', 'vpc': 'vpc01', 'subnets': [ 'subnet01a' ] } rt_ctx2 = { 'name': 'rt02', 'vpc': 'vpc01', 'subnets': [ 'subnet01a' ] } subnet_filters = [{'Name': 'tag:Name', 'Values': ['subnet01a']}] rt01_filters = [{'Name': 'tag:Name', 'Values': ['rt01']}] rt02_filters = [{'Name': 'tag:Name', 'Values': ['rt02']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_rt.RouteTableWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.RouteTable' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(vpc_ctx, self.credentials) h.create_resource() # Create the subnet h = ec2_subnet.create_handler(subnet_ctx, self.credentials) h.create_resource() subnets = list(ec2.subnets.filter(Filters=subnet_filters)) subnet = subnets[0] # Create route table rt01 h = ec2_rt.create_handler(rt_ctx1, self.credentials) h.create_resource() route_tables = list(ec2.route_tables.filter(Filters=rt01_filters)) rt01 = route_tables[0] filters = [ { 'Name': 'association.subnet-id', 'Values': [subnet.id] }, { 'Name': 'association.route-table-id', 'Values': [rt01.id] } ] associations = list(rt01.associations.filter(Filters=filters)) self.assertEqual(len(associations), 1) # Create route table rt02 h = ec2_rt.create_handler(rt_ctx2, self.credentials) h.create_resource() route_tables = list(ec2.route_tables.filter(Filters=rt02_filters)) rt02 = route_tables[0] filters = [ { 'Name': 'association.subnet-id', 'Values': [subnet.id] }, { 'Name': 'association.route-table-id', 'Values': [rt02.id] } ] associations = list(rt02.associations.filter(Filters=filters)) self.assertEqual(len(associations), 1)
class DhcpOptionsHandler(base.BaseHandler): """Manage ``EC2.DhcpOptions`` object lifecycle.""" def __init__(self, ctx, credentials): """Create boto3 session and clients.""" super().__init__() # Create a boto session and add extra features # to the base class EC2.DhcpOptions event = 'creating-resource-class.ec2.DhcpOptions' self._session = Session(**credentials) self._session.events.register(event, _add_wrapper) # boto3 clients self._ec2 = self._session.resource('ec2') self._client = self._session.client('ec2') # Context for the resource we want # to create, update or delete self._ctx = ctx self._name = ctx.get('name') def create_resource(self): """Create a DHCP options set, or update it if it already exists.""" self._resource = self._load( self._name, manager_name='dhcp_options_sets' ) if self._resource: logger.info('DHCP options set %s already exists.', self._name) else: self._create_dhcp_options() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource def update_resource(self): """Update a ``DhcpOptions`` object.""" self._resource = self._load( self._name, manager_name='dhcp_options_sets' ) if self._resource: self._update_dhcp_options() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource else: logger.error('DHCP options set %s does not exist.', self._name) def delete_resource(self): """Delete a ``DhcpOptions`` object.""" self._resource = self._load( self._name, manager_name='dhcp_options_sets' ) if self._resource: logger.info('Removing DHCP options set %s.', self._name) try: self._client.delete_dhcp_options( DhcpOptionsId=self._resource.id ) except ClientError as exc: # If the DHCP options set is associated with a # VPC, we just log the error and ignore it, as # the options set object does not provide any # method to detach itself from the VPC logger.error(exc) else: logger.error('DHCP options set %s does not exist.', self._name) def _create_dhcp_options(self): """Create a ``DhcpOptions`` object.""" dhcp_configurations = [] attrs = ['domain_name_servers', 'domain_name', 'ntp_servers', 'netbios_name_servers', 'netbios_node_type'] for attr in attrs: if attr in self._ctx: dhcp_configurations.append({ 'Key': attr.replace('_', '-'), 'Values': self._ctx.get(attr, []) }) logger.info('Creating DHCP options set %s.', self._name) try: self._resource = self._ec2.create_dhcp_options( DhcpConfigurations=dhcp_configurations ) except (ClientError, ParamValidationError) as exc: logger.error(exc) return # Set the value of the 'Name' tag self._resource.name = self._name # Update other object attributes self._update_dhcp_options() def _update_dhcp_options(self): """Update a ``DhcpOptions`` object.""" self._update_tags()
def __init__(self, instance_id: str): session = Session() self.__instance = session.resource('ec2').Instance(instance_id) self.__ssm = session.client('ssm')
def test_another_internet_gateway_already_attached_2(self): # Case 2: an internet gateway is attached to the VPC and there # is another internet gateway defined in the configuration. We # also leave everything unchanged. igw01_ctx = { 'name': 'igw01' } igw02_ctx = { 'name': 'igw02' } vpc01_ctx1 = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'internet_gateway': 'igw01' } vpc01_ctx2 = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'internet_gateway': 'igw02' } igw01_filters = [{'Name': 'tag:Name', 'Values': ['igw01']}] igw02_filters = [{'Name': 'tag:Name', 'Values': ['igw02']}] vpc01_filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_vpc.VpcWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Vpc' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the first internet gateway h = ec2_igw.create_handler(igw01_ctx, self.credentials) h.create_resource() gateways = list(ec2.internet_gateways.filter(Filters=igw01_filters)) igw01 = gateways[0] self.assertCountEqual(igw01.attachments, []) # Create the VPC h = ec2_vpc.create_handler(vpc01_ctx1, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=vpc01_filters)) vpc01 = vpcs[0] attachments = [{'VpcId': vpc01.id, 'State': 'available'}] # Test that the internet gateway has been attached igw01.reload() self.assertCountEqual(igw01.attachments, attachments) # Create the second internet gateway h = ec2_igw.create_handler(igw02_ctx, self.credentials) h.create_resource() gateways = list(ec2.internet_gateways.filter(Filters=igw02_filters)) igw02 = gateways[0] self.assertCountEqual(igw02.attachments, []) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the VPC with the new context h = ec2_vpc.create_handler(vpc01_ctx2, self.credentials) h.update_resource() # Test that the first internet gateway is still attached to the VPC igw01.reload() self.assertCountEqual(igw01.attachments, attachments) # Test that the second internet gateway is not attached igw02.reload() self.assertCountEqual(igw02.attachments, [])
def test_update_vpc(self): ctx1 = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'tags': { 'description': 'VPC vpc01 (subnet01a & subnet01b)', 'stack': 'Test', 'owner': 'Team A' } } ctx2 = { 'name': 'vpc01', 'cidr_block': '10.0.10.0/24', 'tags': { 'description': 'VPC vpc01 (subnet01a & subnet01b)', 'stack': 'Production' } } tags1 = [ { 'Key': 'Name', 'Value': 'vpc01' }, { 'Key': 'Description', 'Value': 'VPC vpc01 (subnet01a & subnet01b)' }, { 'Key': 'Stack', 'Value': 'Test' }, { 'Key': 'Owner', 'Value': 'Team A' } ] tags2 = [ { 'Key': 'Name', 'Value': 'vpc01' }, { 'Key': 'Description', 'Value': 'VPC vpc01 (subnet01a & subnet01b)' }, { 'Key': 'Stack', 'Value': 'Production' } ] filters = [{'Name': 'tag:Name', 'Values': ['vpc01']}] def _add_wrapper(base_classes, **kwargs): base_classes.insert(0, ec2_vpc.VpcWrapper) with mock_ec2(): event = 'creating-resource-class.ec2.Vpc' session = Session(**self.credentials) session.events.register(event, _add_wrapper) ec2 = session.resource('ec2') # Create the VPC h = ec2_vpc.create_handler(ctx1, self.credentials) h.create_resource() vpcs = list(ec2.vpcs.filter(Filters=filters)) vpc = vpcs[0] vpc_id = vpc.id self.assertEqual(len(vpcs), 1) self.assertCountEqual(vpc.tags, tags1) # We clear the resource cache to simulate a new # program execution with the 'update' option base.BaseHandler._cache.clear() # Update the VPC h = ec2_vpc.create_handler(ctx2, self.credentials) h.update_resource() vpcs = list(ec2.vpcs.filter(Filters=filters)) vpc = vpcs[0] self.assertEqual(len(vpcs), 1) self.assertEqual(vpc.id, vpc_id) self.assertCountEqual(vpc.tags, tags2)
class RouteTableHandler(base.BaseHandler): """Manage ``EC2.RouteTable`` object lifecycle.""" def __init__(self, ctx, credentials): """Create boto3 session and clients.""" super().__init__() # Create a boto session and add extra # features to the base class EC2.RouteTable event = 'creating-resource-class.ec2.RouteTable' self._session = Session(**credentials) self._session.events.register(event, _add_wrapper) # boto3 clients self._ec2 = self._session.resource('ec2') self._client = self._session.client('ec2') # Context for the resource we want # to create, update or delete self._ctx = ctx self._name = ctx.get('name') def create_resource(self): """Create a route table, or update it if it already exists.""" self._resource = self._load(self._name, manager_name='route_tables') if self._resource: logger.info('Route table %s already exists.', self._name) else: self._create_route_table() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource def update_resource(self): """Update a route table object.""" self._resource = self._load(self._name, manager_name='route_tables') if self._resource: self._update_route_table() # Store the object in the cache for future use with self._lock: self._cache[self._name] = self._resource else: logger.error('Route table %s does not exist.', self._name) def delete_resource(self): """Delete a ``RouteTable`` object.""" self._resource = self._load(self._name, manager_name='route_tables') if self._resource: # Remove subnet associations before deleting the route table for subnet_name in self._ctx.get('subnets') or []: subnet = self._load(subnet_name, manager_name='subnets') if subnet: filters = [ { 'Name': 'association.subnet-id', 'Values': [subnet.id] }, { 'Name': 'association.route-table-id', 'Values': [self._resource.id] } ] associations = list(self._resource.associations.filter(Filters=filters)) if associations: # The subnet is associated with this route table try: associations[0].delete() except ClientError as exc: logger.error(exc) # Remove the route table logger.info('Removing route table %s.', self._name) try: self._client.delete_route_table(RouteTableId=self._resource.id) except ClientError as exc: logger.error(exc) else: logger.error('Route table %s does not exist.', self._name) def _create_route_table(self): """Create a ``RouteTable`` object.""" vpc_name = self._ctx['vpc'] vpc = self._load(vpc_name, manager_name='vpcs') if vpc: logger.info('Creating route table %s.', self._name) try: self._resource = self._ec2.create_route_table(VpcId=vpc.id) except ClientError as exc: logger.error(exc) return # Set the value of the 'Name' tag self._resource.name = self._name # Update other object attributes self._update_route_table() else: logger.error( 'Unable to create route table %s because VPC %s does not' ' exist. Operation not performed.', self._name, vpc_name ) def _update_route_table(self): """Update a ``RouteTable`` object.""" self._update_tags() self._update_subnets() self._update_routes() def _update_subnets(self): """Manage subnet associations for the current route table.""" for subnet_name in self._ctx.get('subnets') or []: subnet = self._load(subnet_name, manager_name='subnets') if subnet: # Search if this subnet is already associated with a route table filters = [ { 'Name': 'association.subnet-id', 'Values': [subnet.id] } ] route_tables = list(self._ec2.route_tables.filter(Filters=filters)) if route_tables: if route_tables[0].id == self._resource.id: # The subnet is already associated with this route table continue else: # The subnet is associated with another route table # and we need to update the association associations = list(route_tables[0].associations.filter(Filters=filters)) try: associations[0].replace_subnet(RouteTableId=self._resource.id) logger.info( 'Subnet %s is now associated with route table %s', subnet_name, self._name ) except ClientError as exc: logger.error( 'An error occured while trying to update route table' ' for subnet %s: %s', subnet_name, exc ) else: logger.info( 'Associating subnet %s with route table %s.', subnet_name, self._name ) try: self._resource.associate_with_subnet(SubnetId=subnet.id) except ClientError as exc: logger.error(exc) else: logger.error( 'Unable to associate subnet %s with route table %s because it does' ' not exist. Operation not performed.', subnet_name, self._name ) def _update_routes(self): """Manage routes for the current route table.""" # Routes that are currently set current_routes = self._resource.custom_routes # Routes defined in configuration routes = [] # Routes that do not exist yes new = [] # If a route is already defined for the given CIDR but has # to be updated, we need to use a specific client method modified = [] # Routes that are no longer defined in the configuration trash = [] # Parse the configuration and populate route list for item in self._ctx.get('routes') or []: route = self._build_route(item) if route: routes.append(RouteWrapper(route)) # Search for new routes and routes that should be updated for item in set(routes) - set(current_routes): match = next( (r for r in current_routes if r.destination == item.destination), None ) if match: modified.append(item) else: new.append(item) # Search for routes that should be deleted for item in set(current_routes) - set(routes): match = next( (r for r in routes if r.destination == item.destination), None ) if not match: trash.append(item) for wrapper in modified: logger.info('%s: replacing route - %s.', self._name, wrapper) if all((wrapper.instance_id, wrapper.network_interface_id)): # We must only pass the ``instance_id`` attribute in # order to update a route with an EC2 instance target del wrapper.network_interface_id try: self._client.replace_route(RouteTableId=self._resource.id, **wrapper.route) except (ClientError, ParamValidationError) as exc: logger.error(exc) for wrapper in new: logger.info('%s: creating route - %s.', self._name, wrapper) if all((wrapper.instance_id, wrapper.network_interface_id)): # We must only pass the ``instance_id`` attribute in # order to create a route with an EC2 instance target del wrapper.network_interface_id try: self._resource.create_route(**wrapper.route) except (ClientError, ParamValidationError) as exc: logger.error(exc) for wrapper in trash: logger.info('%s: removing route - %s.', self._name, wrapper) try: self._client.delete_route( RouteTableId=self._resource.id, DestinationCidrBlock=wrapper.destination ) except (ClientError, ParamValidationError) as exc: logger.error(exc) def _build_route(self, item): """Load target objects for route definitions and return a route as a dictionary. The following targets are supported: - internet gateways - virtual private gateways - VPC peering connections - EC2 instances """ filters = [{'Name': 'tag:Name', 'Values': [item['target_name']]}] # internet gateway targets if item['target_type'] == 'internet_gateway': resp = self._client.describe_internet_gateways(Filters=filters) gateways = resp.get('InternetGateways') igw = gateways[0] if gateways else None if igw: return { 'GatewayId': igw['InternetGatewayId'], 'DestinationCidrBlock': item['destination'] } # virtual private gateway targets elif item['target_type'] == 'vpn_gateway': resp = self._client.describe_vpn_gateways(Filters=filters) gateways = resp.get('VpnGateways') vgw = gateways[0] if gateways else None if vgw: return { 'GatewayId': vgw['VpnGatewayId'], 'DestinationCidrBlock': item['destination'] } # VPC peering connection targets elif item['target_type'] == 'vpc_peering_connection': resp = self._client.describe_vpc_peering_connections(Filters=filters) connections = resp.get('VpcPeeringConnections') connection = connections[0] if connections else None if connection: return { 'VpcPeeringConnectionId': connection['VpcPeeringConnectionId'], 'DestinationCidrBlock': item['destination'] } # EC2 instance targets elif item['target_type'] == 'instance': resp = self._client.describe_instances(Filters=filters) reservations = resp.get('Reservations') reservation = reservations[0] if reservations else {} instances = reservation.get('Instances') instance = instances[0] if instances else None if instance: route = { 'InstanceId': instance['InstanceId'], 'DestinationCidrBlock': item['destination'] } interfaces = instance.get('NetworkInterfaces') interface = interfaces[0] if interfaces else None if interface: route['NetworkInterfaceId'] = interface['NetworkInterfaceId'] return route # Unsupported targets else: logger.error("%s: unsupported target type '%s'", self._name, item['target_type']) return None