def modify_role(self, role_info, name, trust_document, policies, attached_policies): changes = list(Differ.compare_two_documents(json.dumps(role_info.assume_role_policy_document), trust_document)) if changes: with self.catch_boto_400("Couldn't modify trust document", "{0} assume document".format(name), trust_document, role=name): for _ in self.change("M", "trust_document", role=name, changes=changes): self.resource.AssumeRolePolicy(name.split('/')[-1]).update(PolicyDocument=trust_document) with self.catch_boto_400("Couldn't get policies for a role", role=name): current_policies = dict((policy.name, policy) for policy in role_info.policies.all()) unknown = [key for key in current_policies if key not in policies] if unknown: log.info("Role has unknown policies that will be disassociated\trole=%s\tunknown=%s", name, unknown) for policy in unknown: with self.catch_boto_400("Couldn't delete a policy from a role", policy=policy, role=name): for _ in self.change("-", "role_policy", role=name, policy=policy): current_policies[policy].delete() for policy, document in policies.items(): has_statements = document and bool(json.loads(document)["Statement"]) if not has_statements: if policy in current_policies: with self.catch_boto_400("Couldn't delete a policy from a role", policy=policy, role=name): for _ in self.change("-", "policy", role=name, policy=policy): current_policies[policy].delete() else: needed = False changes = None if policy in current_policies: changes = list(Differ.compare_two_documents(json.dumps(current_policies.get(policy).policy_document), document)) if changes: log.info("Overriding existing policy\trole=%s\tpolicy=%s", name, policy) needed = True else: log.info("Adding policy to existing role\trole=%s\tpolicy=%s", name, policy) needed = True if needed: with self.catch_boto_400("Couldn't add policy document", "{0} - {1} policy document".format(name, policy), document, role=name, policy=policy): symbol = "M" if changes else "+" for _ in self.change(symbol, "role_policy", role=name, policy=policy, changes=changes, document=document): if policy in current_policies: current_policies[policy].put(PolicyDocument=document) else: self.client.put_role_policy(RoleName=name.split("/")[-1], PolicyName=policy, PolicyDocument=document) self.modify_attached_policies(name, attached_policies)
def modify_route(self, route_info, name, zone, record_type, record_target): old = { "target": route_info['record']['ResourceRecords'], "type": route_info['record']['Type'] } new = {"target": [{"Value": record_target}], 'type': record_type} changes = list(Differ.compare_two_documents(old, new)) hosted_zone_id = self.client.list_hosted_zones_by_name( DNSName=zone)['HostedZones'][0]['Id'] if changes: with self.catch_boto_400("Couldn't change record", record=name, zone=zone): for _ in self.change("M", "record", record=name, zone=zone, changes=changes): self.client.change_resource_record_sets( HostedZoneId=hosted_zone_id, ChangeBatch={ "Changes": [{ "Action": "UPSERT", "ResourceRecordSet": { "Name": "{0}.{1}".format(name, zone), "Type": record_type, "TTL": 60, "ResourceRecords": new['target'] } }] })
def modify_lifecycle(self, bucket_info, name, lifecycle): current = [] with self.ignore_missing(): current_lifecycle = bucket_info.Lifecycle() current_lifecycle.load() current = current_lifecycle.rules new_rules = [] if lifecycle: new_rules = sorted([l.rule for l in lifecycle]) changes = list( Differ.compare_two_documents(json.dumps(current), json.dumps(new_rules))) if changes: with self.catch_boto_400("Couldn't modify lifecycle rules", bucket=name): symbol = "+" if not current else 'M' symbol = '-' if not new_rules else symbol for _ in self.change(symbol, "lifecycle_configuration", bucket=name, changes=changes): if new_rules: current_lifecycle.put( LifecycleConfiguration={"Rules": new_rules}) else: current_lifecycle.put( LifecycleConfiguration={"Rules": []})
def modify_key(self, key_info, name, description, location, grant, policy): client = self.get_client(location) if key_info["Description"] != description: with self.catch_boto_400("Couldn't change the description", alias=name): for _ in self.change("M", "kms_description", alias=name): client.update_key_description(KeyId=key_info["KeyId"], Description=description) changes = list(Differ.compare_two_documents(key_info["Policy"], policy)) if changes: with self.catch_boto_400("Couldn't modify policy", "Key {0} policy".format(name), policy, bucket=name): for _ in self.change("M", "key_policy", alias=name, changes=changes): client.put_key_policy(KeyId=key_info["KeyId"], Policy=policy, PolicyName="default") self.handle_grants(client, key_info["KeyId"], key_info["Grants"], name, [g.statement for g in grant])
def modify_logging(self, bucket_info, name, logging): current_logging = bucket_info.Logging() current = {} with self.ignore_missing(): current_logging.load() if current_logging.logging_enabled is None: current = {} else: current = {"LoggingEnabled": current_logging.logging_enabled} new_document = {} if logging: new_document = logging.document changes = list( Differ.compare_two_documents(json.dumps(current), json.dumps(new_document))) if changes: with self.catch_boto_400("Couldn't modify logging configuration", bucket=name): symbol = "+" if not current else 'M' symbol = '-' if not new_document else symbol for _ in self.change(symbol, "logging_configuration", bucket=name, changes=changes): if new_document: current_logging.put(BucketLoggingStatus=new_document) else: current_logging.put(BucketLoggingStatus={})
def modify_website(self, bucket_info, name, website): current_website = bucket_info.Website() current = {} with self.ignore_missing(): current_website.load() current = { "IndexDocument": current_website.index_document, "ErrorDocument": current_website.error_document, "RedirectAllRequestsTo": current_website.redirect_all_requests_to, "RoutingRules": current_website.routing_rules } current = dict( (key, val) for key, val in current.items() if val is not None) new_document = {} if website: new_document = website.document changes = list( Differ.compare_two_documents(json.dumps(current), json.dumps(new_document))) if changes: with self.catch_boto_400("Couldn't modify website configuration", bucket=name): symbol = "+" if not current else 'M' symbol = '-' if not new_document else symbol for _ in self.change(symbol, "website_configuration", bucket=name, changes=changes): if new_document: current_website.put(WebsiteConfiguration=new_document) else: current_website.delete()
def modify_tags(self, bucket_info, name, tags): current_tags = bucket_info.Tagging() tag_set = [] with self.ignore_missing(): current_tags.load() tag_set = current_tags.tag_set current_vals = dict((tag["Key"], tag["Value"]) for tag in tag_set) changes = list( Differ.compare_two_documents(json.dumps(current_vals), json.dumps(tags))) if changes: new_tag_set = [{ "Value": val, "Key": key } for key, val in tags.items()] with self.catch_boto_400("Couldn't modify tags", bucket=name): symbol = "+" if new_tag_set else "-" symbol = "M" if new_tag_set and current_vals else symbol for _ in self.change(symbol, "bucket_tags", bucket=name, changes=changes): if not new_tag_set: bucket_info.Tagging().delete() else: bucket_info.Tagging().put( Tagging={"TagSet": new_tag_set})
def modify_resource_policy_for_gateway(self, function_arn, function_location, gateway_arn, gateway_name): lambda_client = self.amazon.session.client("lambda", function_location) policy = {} with self.ignore_missing(): policy = lambda_client.get_policy( FunctionName=function_arn)["Policy"] policy = json.loads(policy) statements = policy.get("Statement", []) current_apigateway_statements = [] wanted = { 'Resource': function_arn, 'Effect': 'Allow', 'Action': 'lambda:InvokeFunction', 'Principal': { 'Service': 'apigateway.amazonaws.com' } } for statement in statements: if all(wanted[key] == statement[key] for key in wanted): current_apigateway_statements.append(statement) for current_apigateway_statement in current_apigateway_statements: if current_apigateway_statement.get("Condition", {}).get( "ArnLike", {}).get("AWS:SourceArn") == gateway_arn: # Our work here is done, no changes to make return # At this point, we have no statement allowing our gateway to invoke our lambda function # So let's add a new statement!!! new_statement = wanted new_statement["Condition"] = { "ArnLike": { 'AWS:SourceArn': gateway_arn } } # Make a copy of the statements with our new statement new_statements = list(statements) new_statements.append(new_statement) # Show the differences to the user changes = list(Differ.compare_two_documents(statements, new_statements)) if changes: function_name = function_arn.split(":")[-1] for _ in self.change("M", "Lambda resource policy", gateway=gateway_name, function=function_name, changes=changes): lambda_client.add_permission( FunctionName=function_name, StatementId=str(uuid.uuid1()), Action=new_statement["Action"], Principal=new_statement["Principal"]["Service"], SourceArn=gateway_arn)
def modify_bucket(self, bucket_info, name, permission_document, location, tags): current_location = self.client.get_bucket_location(Bucket=name)['LocationConstraint'] if current_location != location: raise AwsSyncrError("Sorry, can't change the location of a bucket!", wanted=location, currently=current_location, bucket=name) # Make sure we use the correct endpoint to get info from the bucket # So that website buckets don't complain bucket_info.meta.client = self.amazon.session.client("s3", location) bucket_document = "" with self.ignore_missing(): bucket_document = bucket_info.Policy().policy if bucket_document or permission_document: if permission_document and not bucket_document: with self.catch_boto_400("Couldn't add policy", "Bucket {0} policy".format(name), permission_document, bucket=name): for _ in self.change("+", "bucket_policy", bucket=name, changes=list(Differ.compare_two_documents("{}", permission_document))): bucket_info.Policy().put(Policy=permission_document) elif bucket_document and not permission_document: with self.catch_boto_400("Couldn't remove policy", "Bucket {0} policy".format(name), permission_document, bucket=name): for _ in self.change("-", "bucket_policy", bucket=name, changes=list(Differ.compare_two_documents(bucket_document, "{}"))): bucket_info.Policy().delete() else: changes = list(Differ.compare_two_documents(bucket_document, permission_document)) if changes: with self.catch_boto_400("Couldn't modify policy", "Bucket {0} policy".format(name), permission_document, bucket=name): for _ in self.change("M", "bucket_policy", bucket=name, changes=changes): bucket_info.Policy().put(Policy=permission_document) self.modify_tags(bucket_info, name, tags)
def modify_attached_policies(self, role_name, new_policies): """Make sure this role has just the new policies""" parts = role_name.split('/', 1) if len(parts) == 2: prefix, name = parts prefix = "/{0}/".format(prefix) else: prefix = "/" name = parts[0] current_attached_policies = [] with self.ignore_missing(): current_attached_policies = self.client.list_attached_role_policies(RoleName=name) current_attached_policies = [p['PolicyArn'] for p in current_attached_policies["AttachedPolicies"]] new_attached_policies = ["arn:aws:iam::aws:policy/{0}".format(p) for p in new_policies] changes = list(Differ.compare_two_documents(current_attached_policies, new_attached_policies)) if changes: with self.catch_boto_400("Couldn't modify attached policies", role=role_name): for policy in new_attached_policies: if policy not in current_attached_policies: for _ in self.change("+", "attached_policy", role=role_name, policy=policy): self.client.attach_role_policy(RoleName=name, PolicyArn=policy) for policy in current_attached_policies: if policy not in new_attached_policies: for _ in self.change("-", "attached_policy", role=role_name, changes=changes, policy=policy): self.client.detach_role_policy(RoleName=name, PolicyArn=policy)
def modify_resource_method_integration( self, client, gateway_info, location, name, path, method, old_method, new_method, resources_by_path ): old_integration = old_method.get("methodIntegration", {}) new_integration = new_method.integration_request new_kwargs = new_integration.put_kwargs(location, self.accounts, self.environment) old_kwargs = {} if not old_integration else {"type": old_integration["type"]} if old_kwargs and old_kwargs["type"] == "AWS": old_kwargs["uri"] = old_integration["uri"] changes = list(Differ.compare_two_documents(old_kwargs, new_kwargs)) if changes: symbol = "+" if not old_integration else "M" for _ in self.change( symbol, "gateway resource method integration request", gateway=name, resource=path, method=method, type=new_kwargs["type"], changes=changes, ): resource_id = resources_by_path[path]["id"] client.put_integration( restApiId=gateway_info["identity"], resourceId=resource_id, httpMethod=method, integrationHttpMethod=method, **new_kwargs )
def modify_acl(self, bucket_info, name, acl): current_acl = bucket_info.Acl() current_acl.load() current_grants = {"AccessControlPolicy": {"Grants": current_acl.grants}} owner = dict(current_acl.owner) if "ID" or "EmailAddress" in owner: owner["Type"] = "CanonicalUser" else: owner["Type"] = "Group" acl_options = acl(owner) if "ACL" in acl_options: current_grants["ACL"] = acl_options["ACL"] changes = list(Differ.compare_two_documents(json.dumps(current_grants), json.dumps(acl_options))) if changes: with self.catch_boto_400("Couldn't modify acl grants", bucket=name, canned_acl=acl): symbol = "+" if not current_grants else 'M' symbol = '-' if not acl_options else symbol for _ in self.change(symbol, "acl_grants", bucket=name, changes=changes, canned_acl=acl): if "ACL" in acl_options and "AccessControlPolicy" in acl_options: del acl_options["AccessControlPolicy"] else: # owner must be specified # But we don't allow specifying owner in aws_syncr configuration # So, we just set it to the current owner acl_options["AccessControlPolicy"]["Owner"] = current_acl.owner current_acl.put(**acl_options)
def modify_domains(self, client, gateway_info, name, domains): for domain in domains.values(): found = [] matches = [d for d in gateway_info['domains'] if d['domainName'] == domain.full_name] if matches: for mapping in matches[0]['mappings']: if ('identity' in gateway_info and mapping['restApiId'] == gateway_info['identity']) or mapping['basePath'] == domain.base_path: found.append(mapping) current = [dict((key, mapping.get(key)) for key in ('restApiId', 'stage', 'basePath')) for mapping in found] wanted = {'restApiId': gateway_info.get('identity', '<gateway id>'), 'stage': domain.stage, 'basePath': domain.base_path} if list(Differ.compare_two_documents(current, [wanted])): for_removal = [mapping for mapping in current if mapping['basePath'] != wanted['basePath']] for_addition = [mapping for mapping in [wanted] if mapping['basePath'] not in [m['basePath'] for m in found]] for_modification = [] for new in [wanted]: for old in found: if old['basePath'] == new['basePath']: for_modification.append((old, new)) with self.catch_boto_400("Couldn't remove domain name bindings", gateway=name): for mapping in for_removal: for _ in self.change("-", "domain name gateway association", gateway=name, base_path=mapping['basePath']): client.update_base_path_mapping(domainName=domain.full_name, basePath=mapping['basePath'] , patchOperations = [{"op": "remove", "path": "/"}] ) with self.catch_boto_400("Couldn't add domain name bindings", gateway=name): for mapping in for_addition: for _ in self.change("+", "domain name gateway association", gateway=name, base_path=mapping['basePath'], stage=mapping['stage']): client.create_base_path_mapping(domainName=domain.full_name, basePath=mapping['basePath'], restApiId=gateway_info['identity'], stage=mapping['stage']) with self.catch_boto_400("Couldn't modify domain name bindings", gateway=name): for old, new in for_modification: changes = Differ.compare_two_documents(old, new) for _ in self.change("M", "domain name gateway association", gateway=name, stage=new['stage'], base_path=new["basePath"], changes=changes): operations = [] if old['restApiId'] != new['restApiId']: operations.append({"op": "replace", "path": "/restApiId", "value": new['restApiId']}) if old.get('stage') != new.get('stage'): operations.append({"op": "replace", "path": "/stage", "value": new['restApiId']}) client.update_base_path_mapping(domainName=domain.full_name, basePath=wanted['basePath'], patchOperations = operations)
def modify_domains(self, client, gateway_info, name, domains): for domain in domains.values(): found = [] matches = [d for d in gateway_info['domains'] if d['domainName'] == domain.full_name] if matches: for mapping in matches[0]['mappings']: if ('identity' in gateway_info and mapping['restApiId'] == gateway_info['identity']) or mapping['basePath'] == domain.base_path: found.append(mapping) current = [dict((key, mapping.get(key)) for key in ('restApiId', 'stage', 'basePath')) for mapping in found] wanted = {'restApiId': gateway_info.get('identity', '<gateway id>'), 'stage': domain.stage, 'basePath': domain.base_path} if list(Differ.compare_two_documents(current, [wanted])): for_removal = [mapping for mapping in current if mapping['basePath'] != wanted['basePath']] for_addition = [mapping for mapping in [wanted] if mapping['basePath'] not in [m['basePath'] for m in found]] for_modification = [] for new in [wanted]: for old in found: if old['basePath'] == new['basePath']: for_modification.append((old, new)) with self.catch_boto_400("Couldn't remove domain name bindings", gateway=name): for mapping in for_removal: for _ in self.change("-", "domain name gateway association", gateway=name, base_path=mapping['basePath']): client.update_base_path_mapping(domainName=domain.full_name, basePath=mapping['basePath'] , patchOperations = [{"op": "remove", "path": "/"}] ) with self.catch_boto_400("Couldn't add domain name bindings", gateway=name): for mapping in for_addition: for _ in self.change("+", "domain name gateway association", gateway=name, base_path=mapping['basePath'], stage=mapping['stage']): client.create_base_path_mapping(domainName=domain.full_name, basePath=mapping['basePath'], restApiId=gateway_info['identity'], stage=mapping['stage']) with self.catch_boto_400("Couldn't modify domain name bindings", gateway=name): for old, new in for_modification: changes = Differ.compare_two_documents(old, new) for _ in self.change("M", "domain name gateway association", gateway=name, stage=new['stage'], base_path=new["basePath"], changes=changes): operations = [] if old['restApiId'] != new['restApiId']: operations.append({"op": "replace", "path": "/restapiId", "value": new['restApiId']}) if old.get('stage') != new.get('stage'): operations.append({"op": "replace", "path": "/stage", "value": new['restApiId']}) client.update_base_path_mapping(domainName=domain.full_name, basePath=wanted['basePath'], patchOperations = operations)
def create_bucket(self, name, permission_document, bucket): location = bucket.location acl = bucket.acl tags = bucket.tags website = bucket.website logging = bucket.logging lifecycle = bucket.lifecycle bucket = None with self.catch_boto_400("Couldn't Make bucket", bucket=name): for _ in self.change("+", "bucket", bucket=name): bucket = self.resource.create_bucket(Bucket=name, CreateBucketConfiguration={"LocationConstraint": location}) if permission_document: with self.catch_boto_400("Couldn't add policy", "{0} Permission document".format(name), permission_document, bucket=name): for _ in self.change("+", "bucket_policy", bucket=name, document=permission_document): self.resource.Bucket(name).Policy().put(Policy=permission_document) owner = "__owner__" if bucket: owner = bucket.Acl().owner if "ID" or "EmailAddress" in owner: owner["Type"] = "CanonicalUser" else: owner["Type"] = "Group" acl_options = acl(owner) if "ACL" in acl_options: if acl_options["ACL"] != "private": with self.catch_boto_400("Couldn't configure acl", bucket=name, canned_acl=acl): for _ in self.change("+", "acl", bucket=name, acl=acl): self.resource.Bucket(name).BucketAcl().put(ACL=acl) else: with self.catch_boto_400("Couldn't configure acl", bucket=name): for _ in self.change("+", "acl", bucket=name): self.resource.Bucket(name).BucketAcl().put(**acl_options) if website: with self.catch_boto_400("Couldn't add website configuration", bucket=name): for _ in self.change("+", "website_configuration", bucket=name): self.resource.BucketWebsite(name).put(WebsiteConfiguration=website.document) if logging: with self.catch_boto_400("Couldn't add logging configuration", bucket=name): for _ in self.change("+", "logging_configuration", bucket=name): self.resource.BucketLogging(name).put(BucketLoggingStatus=logging.document) if lifecycle: with self.catch_boto_400("Couldn't add logging configuration", bucket=name): for _ in self.change("+", "lifecycle_configuration", bucket=name): self.resource.BucketLifecycle(name).put(LifecycleConfiguration=sorted(l.rule for l in lifecycle)) if tags: with self.catch_boto_400("Couldn't add tags", bucket=name): tag_set = [{"Value": val, "Key": key} for key, val in tags.items()] changes = list(Differ.compare_two_documents("[]", json.dumps(tag_set))) for _ in self.change("+", "bucket_tags", bucket=name, tags=tags, changes=changes): self.resource.Bucket(name).Tagging().put(Tagging={"TagSet": tag_set})
def modify_resource_method_integration_response(self, client, gateway_info, name, path, method, old_method, new_method, resources_by_path): old_integration = old_method.get('methodIntegration', {}).get("integrationResponses", {}) wanted_integration = dict((str(s), v) for s, v in new_method.integration_response.responses.items()) for_removal = set(old_integration) - set(wanted_integration) for_addition = set(wanted_integration) - set(old_integration) for_modification = [s for s in wanted_integration if s in old_integration] for status_code in for_removal: for _ in self.change("-", "gateway resource integration response", gateway=name, resource=path, method=method, status_code=status_code): resource_id = resources_by_path[path]['id'] client.delete_integration_response(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method, statusCode=str(status_code)) for status_code in for_addition: for _ in self.change("+", "gateway resource integration response", gateway=name, resource=path, method=method, status_code=status_code): resource_id = resources_by_path[path]['id'] client.put_integration_response(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method, statusCode=str(status_code) , responseTemplates = {} if not wanted_integration[status_code] else dict((m.content_type, m.template) for m in wanted_integration[status_code]) ) # Modify response Templates for status_code in for_modification: old = old_integration[status_code].get('responseTemplates', {}) for ct, template in old.items(): if template is None: old[ct] = "" new = {} if wanted_integration[status_code]: new = dict((m.content_type, m.template) for m in wanted_integration[status_code]) changes = list(Differ.compare_two_documents(old, new)) old = dict((ct.replace('/', '~1'), v) for ct, v in old.items()) new = dict((ct.replace('/', '~1'), v) for ct, v in new.items()) if changes: for_removal = set(old) - set(new) for_addition = set(new) - set(old) for_mod = [content_type for content_type in new if content_type in old] operations = [] for content_type in for_removal: operations.append({"op": "remove", "path": "/responseTemplates/{0}".format(content_type)}) for content_type in for_addition: operations.append({"op": "add", "path":"/responseTemplates/{0}".format(content_type), 'value': new[content_type]}) for content_type in for_mod: operations.append({"op": "replace", "path":"/responseTemplates/{0}".format(content_type), 'value': new[content_type]}) for _ in self.change("M", "gateway resource integration response", gateway=name, resource=path, method=method, status_code=status_code, changes=changes): resource_id = resources_by_path[path]['id'] client.update_integration_response(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method, statusCode=str(status_code) , patchOperations = operations )
def modify_resource_method_status_codes(self, client, gateway_info, name, path, method, old_method, new_method, resources_by_path): old_status_codes = list(old_method.get('methodResponses', {}).keys()) new_status_codes = list(str(st) for st in new_method.method_response.responses.keys()) for_removal = set(old_status_codes) - set(new_status_codes) for_addition = set(new_status_codes) - set(old_status_codes) for_modification = [status_code for status_code in new_status_codes if status_code in old_status_codes] for status_code in for_removal: for _ in self.change("-", "gateway resource method response", gateway=name, resource=path, method=method, status_code=status_code): resource_id = resources_by_path[path]['id'] client.delete_method_response(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method, statusCode=str(status_code)) for status_code in for_addition: for _ in self.change("+", "gateway resource method response", gateway=name, resource=path, method=method, status_code=status_code): resource_id = resources_by_path[path]['id'] models = {new_method.method_response.responses[int(status_code)]: "Empty"} models = dict((ct, model) for ct, model in models.items() if ct != "application/json") client.put_method_response(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method, statusCode=str(status_code) , responseParameters = {} , responseModels = models ) for status_code in for_modification: new = {new_method.method_response.responses[int(status_code)]: "Empty"} new = dict((ct, model) for ct, model in new.items() if ct != "application/json") old = old_method["methodResponses"][status_code].get("responseModels", {}) changes = list(Differ.compare_two_documents(old, new)) old = dict((ct.replace('/', '~1'), v) for ct, v in old.items()) new = dict((ct.replace('/', '~1'), v) for ct, v in new.items()) if changes: for_removal = set(old) - set(new) for_addition = set(new) - set(old) for_mod = [content_type for content_type in new if content_type in old] operations = [] for content_type in for_removal: operations.append({"op": "remove", "path": "/responseModels/{0}".format(content_type)}) for content_type in for_addition: operations.append({"op": "add", "path":"/responseModels/{0}".format(content_type), 'value': new[content_type]}) for content_type in for_mod: operations.append({"op": "replace", "path":"/responseModels/{0}".format(content_type), 'value': new[content_type]}) for _ in self.change("M", "gateway resource method response model", gateway=name, resource=path, method=method, status_code=status_code, changes=changes): resource_id = resources_by_path[path]['id'] client.update_method_response(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method, statusCode=str(status_code) , patchOperations = operations )
def modify_resource_policy_for_gateway(self, function_arn, function_location, gateway_arn, gateway_name): lambda_client = self.amazon.session.client("lambda", function_location) policy = {} with self.ignore_missing(): policy = lambda_client.get_policy(FunctionName=function_arn)["Policy"] policy = json.loads(policy) statements = policy.get("Statement", []) current_apigateway_statements = [] wanted = { "Resource": function_arn, "Effect": "Allow", "Action": "lambda:InvokeFunction", "Principal": {"Service": "apigateway.amazonaws.com"}, } for statement in statements: if all(wanted[key] == statement[key] for key in wanted): current_apigateway_statements.append(statement) for current_apigateway_statement in current_apigateway_statements: if current_apigateway_statement.get("Condition", {}).get("ArnLike", {}).get("AWS:SourceArn") == gateway_arn: # Our work here is done, no changes to make return # At this point, we have no statement allowing our gateway to invoke our lambda function # So let's add a new statement!!! new_statement = wanted new_statement["Condition"] = {"ArnLike": {"AWS:SourceArn": gateway_arn}} # Make a copy of the statements with our new statement new_statements = list(statements) new_statements.append(new_statement) # Show the differences to the user changes = list(Differ.compare_two_documents(statements, new_statements)) if changes: function_name = function_arn.split(":")[-1] for _ in self.change( "M", "Lambda resource policy", gateway=gateway_name, function=function_name, changes=changes ): lambda_client.add_permission( FunctionName=function_name, StatementId=str(uuid.uuid1()), Action=new_statement["Action"], Principal=new_statement["Principal"]["Service"], SourceArn=gateway_arn, )
def create_bucket(self, name, permission_document, location, tags): with self.catch_boto_400("Couldn't Make bucket", bucket=name): for _ in self.change("+", "bucket", bucket=name): self.resource.create_bucket(Bucket=name, CreateBucketConfiguration={"LocationConstraint": location}) if permission_document: with self.catch_boto_400("Couldn't add policy", "{0} Permission document".format(name), permission_document, bucket=name): for _ in self.change("+", "bucket_policy", bucket=name, document=permission_document): self.resource.Bucket(name).Policy().put(Policy=permission_document) if tags: with self.catch_boto_400("Couldn't add tags", bucket=name): tag_set = [{"Value": val, "Key": key} for key, val in tags.items()] changes = list(Differ.compare_two_documents("[]", json.dumps(tag_set))) for _ in self.change("+", "bucket_tags", bucket=name, tags=tags, changes=changes): self.resource.Bucket(name).Tagging().put(Tagging={"TagSet": tag_set})
def modify_api_keys(self, client, gateway_info, name, api_keys): current = [ak["name"] for ak in gateway_info["api_keys"]] wanted = [api_key.name for api_key in api_keys] for_addition = list(set(wanted) - set(current)) for keyname in for_addition: with self.catch_boto_400("Couldn't add api keys", api_key=keyname): for _ in self.change("+", "gateway api key", gateway=name, api_key=keyname): api_key = [api_key for api_key in api_keys if api_key.name == keyname][0] client.create_api_key( name=keyname, enabled=True, stageKeys=[ {"restApiId": gateway_info["identity"], "stageName": stage} for stage in api_key.stages ], ) for_modification = [key for key in current if key in wanted] for keyname in for_modification: with self.catch_boto_400("Couldn't modify api keys", api_key=keyname): api_key = [api_key for api_key in api_keys if api_key.name == keyname][0] old_api_key = [api_key for api_key in gateway_info["api_keys"] if api_key["name"] == keyname][0] other_api_stages = [ key for key in old_api_key["stageKeys"] if key[: key.find("/")] != gateway_info["identity"] ] operations = [] new_stage_keys = [ "{0}/{1}".format(gateway_info["identity"], key) for key in api_key.stages ] + other_api_stages changes = list(Differ.compare_two_documents(sorted(old_api_key["stageKeys"]), sorted(new_stage_keys))) if changes: for_removal = set(old_api_key["stageKeys"]) - set(new_stage_keys) for key in for_removal: operations.append({"op": "remove", "path": "/stages", "value": key}) for_addition = set(new_stage_keys) - set(old_api_key["stageKeys"]) for key in for_addition: operations.append({"op": "add", "path": "/stages", "value": key}) if operations: for _ in self.change("M", "gateway api key", gateway=name, api_key=keyname, changes=changes): client.update_api_key(apiKey=old_api_key["id"], patchOperations=operations)
def modify_function(self, function_info, name, description, location, runtime, role, handler, timeout, memory_size, code): client = self.amazon.session.client('lambda', location) wanted = dict( FunctionName=name, Role=role, Handler=handler , Description=description, Timeout=timeout, MemorySize=memory_size ) current = dict((key, function_info["Configuration"][key]) for key in ( "FunctionName", "Role", "Handler", "Description", "Timeout", "MemorySize" ) ) changes = list(Differ.compare_two_documents(current, wanted)) if changes: with self.catch_boto_400("Couldn't modify function", function=name): for _ in self.change("M", "function", changes=changes, function=name): client.update_function_configuration(**wanted)
def modify_resource_method_integration(self, client, gateway_info, location, name, path, method, old_method, new_method, resources_by_path): old_integration = old_method.get('methodIntegration', {}) new_integration = new_method.integration_request new_kwargs = new_integration.put_kwargs(location, self.accounts, self.environment) old_kwargs = {} if not old_integration else {"type": old_integration["type"]} if old_kwargs and old_kwargs['type'] == 'AWS': old_kwargs['uri'] = old_integration['uri'] changes = list(Differ.compare_two_documents(old_kwargs, new_kwargs)) if changes: symbol = "+" if not old_integration else 'M' for _ in self.change(symbol, "gateway resource method integration request", gateway=name, resource=path, method=method, type=new_kwargs['type'], changes=changes): resource_id = resources_by_path[path]['id'] client.put_integration(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method , integrationHttpMethod=method , **new_kwargs )
def modify_acl(self, bucket_info, name, acl): current_acl = bucket_info.Acl() current_acl.load() current_grants = { "AccessControlPolicy": { "Grants": current_acl.grants } } owner = dict(current_acl.owner) if "ID" or "EmailAddress" in owner: owner["Type"] = "CanonicalUser" else: owner["Type"] = "Group" acl_options = acl(owner) if "ACL" in acl_options: current_grants["ACL"] = acl_options["ACL"] changes = list( Differ.compare_two_documents(json.dumps(current_grants), json.dumps(acl_options))) if changes: with self.catch_boto_400("Couldn't modify acl grants", bucket=name, canned_acl=acl): symbol = "+" if not current_grants else 'M' symbol = '-' if not acl_options else symbol for _ in self.change(symbol, "acl_grants", bucket=name, changes=changes, canned_acl=acl): if "ACL" in acl_options and "AccessControlPolicy" in acl_options: del acl_options["AccessControlPolicy"] else: # owner must be specified # But we don't allow specifying owner in aws_syncr configuration # So, we just set it to the current owner acl_options["AccessControlPolicy"][ "Owner"] = current_acl.owner current_acl.put(**acl_options)
def modify_resource_method_integration(self, client, gateway_info, location, name, path, method, old_method, new_method, resources_by_path): old_integration = old_method.get('methodIntegration', {}) new_integration = new_method.integration_request new_kwargs = new_integration.put_kwargs(location, self.accounts, self.environment) old_kwargs = {} if not old_integration else {"type": old_integration["type"], "httpMethod": old_integration.get("httpMethod")} if old_integration and old_integration.get('requestTemplates'): old_kwargs['requestTemplates'] = old_integration['requestTemplates'] for ct, template in list(old_kwargs['requestTemplates'].items()): if not template: old_kwargs['requestTemplates'][ct] = "" if old_kwargs and old_kwargs['type'] == 'AWS': old_kwargs['uri'] = old_integration['uri'] elif old_kwargs and old_kwargs['type'] == 'MOCK': old_kwargs["httpMethod"] = method changes = list(Differ.compare_two_documents(old_kwargs, new_kwargs)) # Make sure our integration can be called by apigateway if 'identity' in gateway_info: arn = "arn:aws:execute-api:{0}:{1}:{2}/*/".format(location, self.account_id, gateway_info['identity']) new_integration.create_permissions(self.amazon, arn, name, self.accounts, self.environment) else: # Only possible in dry-run new_integration.announce_create_permissions(name, self.change) if changes: symbol = "+" if not old_integration else 'M' for _ in self.change(symbol, "gateway resource method integration request", gateway=name, resource=path, method=method, type=new_kwargs['type'], changes=changes): resource_id = resources_by_path[path]['id'] integration_method = new_kwargs.pop("httpMethod") res = client.put_integration(restApiId=gateway_info['identity'], resourceId=resource_id, httpMethod=method , integrationHttpMethod=integration_method , **new_kwargs ) # put_integration removes the integration response so we take it away from our record # And let modify_resource_method_integration_response deal with the dissapearance if 'integrationResponses' in old_method.get("methodIntegration", {}): del old_method["methodIntegration"]['integrationResponses'] new_kwargs['responseTemplates'] = new_method.integration_response.responses.items()
def modify_tags(self, bucket_info, name, tags): current_tags = bucket_info.Tagging() tag_set = [] with self.ignore_missing(): current_tags.load() tag_set = current_tags.tag_set current_vals = dict((tag["Key"], tag["Value"]) for tag in tag_set) changes = list(Differ.compare_two_documents(json.dumps(current_vals), json.dumps(tags))) if changes: new_tag_set = [{"Value": val, "Key": key} for key, val in tags.items()] with self.catch_boto_400("Couldn't modify tags", bucket=name): symbol = "+" if new_tag_set else "-" symbol = "M" if new_tag_set and current_vals else symbol for _ in self.change(symbol, "bucket_tags", bucket=name, changes=changes): if not new_tag_set: bucket_info.Tagging().delete() else: bucket_info.Tagging().put(Tagging={"TagSet": new_tag_set})
def create_route(self, name, zone, record_type, record_target): old = {} new = {"target": [{"Value": record_target}], 'type': record_type} changes = list(Differ.compare_two_documents(old, new)) hosted_zone_id = self.client.list_hosted_zones_by_name(DNSName=zone)['HostedZones'][0]['Id'] with self.catch_boto_400("Couldn't add record", record=name, zone=zone): for _ in self.change("+", "record", record=name, zone=zone, changes=changes): self.client.change_resource_record_sets(HostedZoneId=hosted_zone_id , ChangeBatch = {"Changes": [ { "Action": "CREATE" , "ResourceRecordSet": { "Name": "{0}.{1}".format(name, zone) , "Type": record_type , "TTL": 60 , "ResourceRecords": new['target'] } } ] } )
def modify_lifecycle(self, bucket_info, name, lifecycle): current = [] with self.ignore_missing(): current_lifecycle = bucket_info.Lifecycle() current_lifecycle.load() current = current_lifecycle.rules new_rules = [] if lifecycle: new_rules = sorted([l.rule for l in lifecycle]) changes = list(Differ.compare_two_documents(json.dumps(current), json.dumps(new_rules))) if changes: with self.catch_boto_400("Couldn't modify lifecycle rules", bucket=name): symbol = "+" if not current else 'M' symbol = '-' if not new_rules else symbol for _ in self.change(symbol, "lifecycle_configuration", bucket=name, changes=changes): if new_rules: current_lifecycle.put(LifecycleConfiguration={"Rules": new_rules}) else: current_lifecycle.put(LifecycleConfiguration={"Rules": []})
def modify_website(self, bucket_info, name, website): current_website = bucket_info.Website() current = {} with self.ignore_missing(): current_website.load() current = {"IndexDocument": current_website.index_document, "ErrorDocument": current_website.error_document, "RedirectAllRequestsTo": current_website.redirect_all_requests_to, "RoutingRules": current_website.routing_rules} current = dict((key, val) for key, val in current.items() if val is not None) new_document = {} if website: new_document = website.document changes = list(Differ.compare_two_documents(json.dumps(current), json.dumps(new_document))) if changes: with self.catch_boto_400("Couldn't modify website configuration", bucket=name): symbol = "+" if not current else 'M' symbol = '-' if not new_document else symbol for _ in self.change(symbol, "website_configuration", bucket=name, changes=changes): if new_document: current_website.put(WebsiteConfiguration=new_document) else: current_website.delete()
def modify_api_keys(self, client, gateway_info, name, api_keys): current = [ak['name'] for ak in gateway_info['api_keys']] wanted = [api_key.name for api_key in api_keys] for_addition = list(set(wanted) - set(current)) for keyname in for_addition: with self.catch_boto_400("Couldn't add api keys", api_key=keyname): for _ in self.change("+", "gateway api key", gateway=name, api_key=keyname): api_key = [api_key for api_key in api_keys if api_key.name == keyname][0] client.create_api_key(name=keyname, enabled=True , stageKeys=[{'restApiId': gateway_info['identity'], 'stageName': stage} for stage in api_key.stages] ) for_modification = [key for key in current if key in wanted] for keyname in for_modification: with self.catch_boto_400("Couldn't modify api keys", api_key=keyname): api_key = [api_key for api_key in api_keys if api_key.name == keyname][0] old_api_key = [api_key for api_key in gateway_info['api_keys'] if api_key['name'] == keyname][0] other_api_stages = [key for key in old_api_key['stageKeys'] if key[:key.find('/')] != gateway_info['identity']] operations = [] new_stage_keys = ["{0}/{1}".format(gateway_info['identity'], key) for key in api_key.stages] + other_api_stages changes = list(Differ.compare_two_documents(sorted(old_api_key['stageKeys']), sorted(new_stage_keys))) if changes: for_removal = set(old_api_key['stageKeys']) - set(new_stage_keys) for key in for_removal: operations.append({"op": "remove", "path": "/stages", "value": key}) for_addition = set(new_stage_keys) - set(old_api_key['stageKeys']) for key in for_addition: operations.append({"op": "add", "path": "/stages", "value": key}) if operations: for _ in self.change("M", "gateway api key", gateway=name, api_key=keyname, changes=changes): client.update_api_key(apiKey=old_api_key['id'], patchOperations=operations)
def modify_function(self, function_info, name, description, location, runtime, role, handler, timeout, memory_size, code): client = self.amazon.session.client('lambda', location) wanted = dict(FunctionName=name, Role=role, Handler=handler, Description=description, Timeout=timeout, MemorySize=memory_size) current = dict((key, function_info["Configuration"][key]) for key in ("FunctionName", "Role", "Handler", "Description", "Timeout", "MemorySize")) changes = list(Differ.compare_two_documents(current, wanted)) if changes: with self.catch_boto_400("Couldn't modify function", function=name): for _ in self.change("M", "function", changes=changes, function=name): client.update_function_configuration(**wanted)
def modify_logging(self, bucket_info, name, logging): current_logging = bucket_info.Logging() current = {} with self.ignore_missing(): current_logging.load() if current_logging.logging_enabled is None: current = {} else: current = {"LoggingEnabled": current_logging.logging_enabled} new_document = {} if logging: new_document = logging.document changes = list(Differ.compare_two_documents(json.dumps(current), json.dumps(new_document))) if changes: with self.catch_boto_400("Couldn't modify logging configuration", bucket=name): symbol = "+" if not current else 'M' symbol = '-' if not new_document else symbol for _ in self.change(symbol, "logging_configuration", bucket=name, changes=changes): if new_document: current_logging.put(BucketLoggingStatus=new_document) else: current_logging.put(BucketLoggingStatus={})
def create_bucket(self, name, permission_document, bucket): location = bucket.location acl = bucket.acl tags = bucket.tags website = bucket.website logging = bucket.logging lifecycle = bucket.lifecycle bucket = None with self.catch_boto_400("Couldn't Make bucket", bucket=name): for _ in self.change("+", "bucket", bucket=name): bucket = self.resource.create_bucket( Bucket=name, CreateBucketConfiguration={"LocationConstraint": location}) if permission_document: with self.catch_boto_400("Couldn't add policy", "{0} Permission document".format(name), permission_document, bucket=name): for _ in self.change("+", "bucket_policy", bucket=name, document=permission_document): self.resource.Bucket(name).Policy().put( Policy=permission_document) owner = "__owner__" if bucket: owner = bucket.Acl().owner if "ID" or "EmailAddress" in owner: owner["Type"] = "CanonicalUser" else: owner["Type"] = "Group" acl_options = acl(owner) if "ACL" in acl_options: if acl_options["ACL"] != "private": with self.catch_boto_400("Couldn't configure acl", bucket=name, canned_acl=acl): for _ in self.change("+", "acl", bucket=name, acl=acl): self.resource.Bucket(name).BucketAcl().put(ACL=acl) else: with self.catch_boto_400("Couldn't configure acl", bucket=name): for _ in self.change("+", "acl", bucket=name): self.resource.Bucket(name).BucketAcl().put(**acl_options) if website: with self.catch_boto_400("Couldn't add website configuration", bucket=name): for _ in self.change("+", "website_configuration", bucket=name): self.resource.BucketWebsite(name).put( WebsiteConfiguration=website.document) if logging: with self.catch_boto_400("Couldn't add logging configuration", bucket=name): for _ in self.change("+", "logging_configuration", bucket=name): self.resource.BucketLogging(name).put( BucketLoggingStatus=logging.document) if lifecycle: with self.catch_boto_400("Couldn't add logging configuration", bucket=name): for _ in self.change("+", "lifecycle_configuration", bucket=name): self.resource.BucketLifecycle(name).put( LifecycleConfiguration=sorted(l.rule for l in lifecycle)) if tags: with self.catch_boto_400("Couldn't add tags", bucket=name): tag_set = [{ "Value": val, "Key": key } for key, val in tags.items()] changes = list( Differ.compare_two_documents("[]", json.dumps(tag_set))) for _ in self.change("+", "bucket_tags", bucket=name, tags=tags, changes=changes): self.resource.Bucket(name).Tagging().put( Tagging={"TagSet": tag_set})
def modify_role(self, role_info, name, trust_document, policies): changes = list( Differ.compare_two_documents( json.dumps(role_info.assume_role_policy_document), trust_document)) if changes: with self.catch_boto_400("Couldn't modify trust document", "{0} assume document".format(name), trust_document, role=name): for _ in self.change("M", "trust_document", role=name, changes=changes): self.resource.AssumeRolePolicy(name.split('/')[-1]).update( PolicyDocument=trust_document) with self.catch_boto_400("Couldn't get policies for a role", role=name): current_policies = dict( (policy.name, policy) for policy in role_info.policies.all()) unknown = [key for key in current_policies if key not in policies] if unknown: log.info( "Role has unknown policies that will be disassociated\trole=%s\tunknown=%s", name, unknown) for policy in unknown: with self.catch_boto_400( "Couldn't delete a policy from a role", policy=policy, role=name): for _ in self.change("-", "role_policy", role=name, policy=policy): current_policies[policy].delete() for policy, document in policies.items(): if not document: if policy in current_policies: with self.catch_boto_400( "Couldn't delete a policy from a role", policy=policy, role=name): for _ in self.change("-", "policy", role=name, policy=policy): current_policies[policy].delete() else: needed = False changes = None if policy in current_policies: changes = list( Differ.compare_two_documents( json.dumps( current_policies.get(policy).policy_document), document)) if changes: log.info( "Overriding existing policy\trole=%s\tpolicy=%s", name, policy) needed = True else: log.info( "Adding policy to existing role\trole=%s\tpolicy=%s", name, policy) needed = True if needed: with self.catch_boto_400( "Couldn't add policy document", "{0} - {1} policy document".format(name, policy), document, role=name, policy=policy): symbol = "M" if changes else "+" for _ in self.change(symbol, "role_policy", role=name, policy=policy, changes=changes, document=document): current_policies[policy].put( PolicyDocument=document)
def modify_domains(self, client, gateway_info, name, domains): for domain in domains.values(): found = [] matches = [d for d in gateway_info["domains"] if d["domainName"] == domain.full_name] if matches: for mapping in matches[0]["mappings"]: if ("identity" in gateway_info and mapping["restApiId"] == gateway_info["identity"]) or mapping[ "basePath" ] == domain.base_path: found.append(mapping) current = [dict((key, mapping.get(key)) for key in ("restApiId", "stage", "basePath")) for mapping in found] wanted = { "restApiId": gateway_info.get("identity", "<gateway id>"), "stage": domain.stage, "basePath": domain.base_path, } if list(Differ.compare_two_documents(current, [wanted])): for_removal = [mapping for mapping in current if mapping["basePath"] != wanted["basePath"]] for_addition = [ mapping for mapping in [wanted] if mapping["basePath"] not in [m["basePath"] for m in found] ] for_modification = [] for new in [wanted]: for old in found: if old["basePath"] == new["basePath"]: for_modification.append((old, new)) with self.catch_boto_400("Couldn't remove domain name bindings", gateway=name): for mapping in for_removal: for _ in self.change( "-", "domain name gateway association", gateway=name, base_path=mapping["basePath"] ): client.update_base_path_mapping( domainName=domain.full_name, basePath=mapping["basePath"], patchOperations=[{"op": "remove", "path": "/"}], ) with self.catch_boto_400("Couldn't add domain name bindings", gateway=name): for mapping in for_addition: for _ in self.change( "+", "domain name gateway association", gateway=name, base_path=mapping["basePath"], stage=mapping["stage"], ): client.create_base_path_mapping( domainName=domain.full_name, basePath=mapping["basePath"], restApiId=gateway_info["identity"], stage=mapping["stage"], ) with self.catch_boto_400("Couldn't modify domain name bindings", gateway=name): for old, new in for_modification: changes = Differ.compare_two_documents(old, new) for _ in self.change( "M", "domain name gateway association", gateway=name, stage=new["stage"], base_path=new["basePath"], changes=changes, ): operations = [] if old["restApiId"] != new["restApiId"]: operations.append({"op": "replace", "path": "/restApiId", "value": new["restApiId"]}) if old.get("stage") != new.get("stage"): operations.append({"op": "replace", "path": "/stage", "value": new["restApiId"]}) client.update_base_path_mapping( domainName=domain.full_name, basePath=wanted["basePath"], patchOperations=operations )