def get_object(**kwargs): client = kwargs.get("client") assume_role = kwargs.get("assume_role") if not client: if assume_role: client = boto3_cached_conn( "s3", account_number=kwargs.get("account_number"), assume_role=assume_role, session_name=kwargs.get("session_name", "ConsoleMe"), region=kwargs.get("region", config.region), retry_max_attempts=2, client_kwargs=config.get("boto3.client_kwargs", {}), ) else: client = boto3.client("s3", **config.get("boto3.client_kwargs", {})) return client.get_object(Bucket=kwargs.get("Bucket"), Key=kwargs.get("Key"))
def get(self, group=None): user = DynamoHandler().get_requests_by_user("nag") resource = boto3_cached_conn( 'dynamodb', service_type='resource') print (resource) table = resource.Table("pet") if not group: response = table.scan() #response = table.get_item(Key={"par": "test"}) self.write("Response {} ".format(json.dumps(response["Items"], indent=1))) else: print(group) print("00000") response = table.get_item(Key={"par": group}) print(response) if response.get('Item', None): print(dir(response)) self.write(" Response {} ".format(json.dumps(response["Item"],indent=1))) else: self.write("User :{} Not found".format(group))
def _get_arns(self): """ Gets a list of all Role ARNs in a given account, optionally limited by class property ARN filter :return: list of role ARNs """ client = boto3_cached_conn('iam', service_type='client', **self.conn_details) account_arns = set() for role in list_roles(**self.conn_details): account_arns.add(role['Arn']) for user in list_users(**self.conn_details): account_arns.add(user['Arn']) for page in client.get_paginator('list_policies').paginate( Scope='Local'): for policy in page['Policies']: account_arns.add(policy['Arn']) for page in client.get_paginator('list_groups').paginate(): for group in page['Groups']: account_arns.add(group['Arn']) result_arns = set() for arn in self.arn_list: if arn.lower() == 'all': return account_arns if arn not in account_arns: self.current_app.logger.warn( "Provided ARN {arn} not found in account.".format(arn=arn)) continue result_arns.add(arn) return list(result_arns)
def get_enabled_regions_for_account(account_id: str) -> Set[str]: """ Returns a list of regions enabled for an account based on an EC2 Describe Regions call. Can be overridden with a global configuration of static regions (Configuration key: `celery.sync_regions`), or a configuration of specific regions per account (Configuration key: `get_enabled_regions_for_account.{account_id}`) """ enabled_regions_for_account = config.get( f"get_enabled_regions_for_account.{account_id}") if enabled_regions_for_account: return enabled_regions_for_account celery_sync_regions = config.get("celery.sync_regions", []) if celery_sync_regions: return celery_sync_regions client = boto3_cached_conn( "ec2", account_number=account_id, assume_role=config.get("policies.role_name"), read_only=True, ) return set(r["RegionName"] for r in client.describe_regions()["Regions"])
def mock_classic_link(ec2): conn = boto3_cached_conn("ec2", service_type="client", future_expiration_minutes=15, account_number="012345678912", region='us-east-1') def describe_vpc_classic_link(**kwargs): """ # Too lazy to submit a PR to Moto -- not worth the trouble for ClassicLink :param kwargs: :return: """ return { "Vpcs": [{ "ClassicLinkEnabled": True, "Tags": [], "VpcId": kwargs["VpcIds"][0] }] } def describe_vpc_classic_link_dns_support(**kwargs): """ # Too lazy to submit a PR to Moto -- not worth the trouble for ClassicLink :param kwargs: :return: """ return { "Vpcs": [{ "ClassicLinkDnsSupported": True, "VpcId": kwargs["VpcIds"][0] }] } setattr(conn, "describe_vpc_classic_link", describe_vpc_classic_link) setattr(conn, "describe_vpc_classic_link_dns_support", describe_vpc_classic_link_dns_support) return conn
def apply_managed_policy_to_role(role: Dict, policy_name: str, session_name: str) -> bool: """ Apply a managed policy to a role. :param role: An AWS role dictionary (from a boto3 get_role or get_account_authorization_details call) :param policy_name: Name of managed policy to add to role :param session_name: Name of session to assume role with. This is an identifier that will be logged in CloudTrail :return: """ function = f"{__name__}.{sys._getframe().f_code.co_name}" log_data = { "function": function, "role": role, "policy_name": policy_name, "session_name": session_name, } account_id = role.get("Arn").split(":")[4] policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}" client = boto3_cached_conn( "iam", account_number=account_id, assume_role=config.get("policies.role_name"), session_name=session_name, ) client.attach_role_policy(RoleName=role.get("RoleName"), PolicyArn=policy_arn) log_data["message"] = "Applied managed policy to role" log.debug(log_data) stats.count( f"{function}.attach_role_policy", tags={ "role": role.get("Arn"), "policy": policy_arn }, ) return True
def update_repoed_description(role_name: str, conn_details: Dict[str, Any]) -> None: client: IAMClient = boto3_cached_conn("iam", **conn_details) try: description = client.get_role(RoleName=role_name)["Role"].get("Description", "") except KeyError: return date_string = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%m/%d/%y") if "; Repokid repoed" in description: new_description = re.sub( r"; Repokid repoed [0-9]{2}\/[0-9]{2}\/[0-9]{2}", f"; Repokid repoed {date_string}", description, ) else: new_description = description + " ; Repokid repoed {}".format(date_string) # IAM role descriptions have a max length of 1000, if our new length would be longer, skip this if len(new_description) < 1000: client.update_role_description(RoleName=role_name, Description=new_description) else: LOGGER.error( "Unable to set repo description ({}) for role {}, length would be too long".format( new_description, role_name ) )
def query( query: str, use_aggregator: bool = True, account_id: Optional[str] = None ) -> List: resources = [] if use_aggregator: config_client = boto3.client("config", region_name=config.region) configuration_aggregator_name: str = config.get( "aws_config.configuration_aggregator.name" ).format(region=config.region) if not configuration_aggregator_name: raise MissingConfigurationValue("Invalid configuration for aws_config") response = config_client.select_aggregate_resource_config( Expression=query, ConfigurationAggregatorName=configuration_aggregator_name, Limit=100, ) for r in response.get("Results", []): resources.append(json.loads(r)) while response.get("NextToken"): response = config_client.select_aggregate_resource_config( Expression=query, ConfigurationAggregatorName=configuration_aggregator_name, Limit=100, NextToken=response["NextToken"], ) for r in response.get("Results", []): resources.append(json.loads(r)) return resources else: # Don't use Config aggregator and instead query all the regions on an account session = boto3.Session() available_regions = session.get_available_regions("config") excluded_regions = config.get( "api_protect.exclude_regions", ["af-south-1", "ap-east-1", "ap-northeast-3", "eu-south-1", "me-south-1"], ) regions = [x for x in available_regions if x not in excluded_regions] for region in regions: config_client = boto3_cached_conn( "config", account_number=account_id, assume_role=config.get("policies.role_name"), region=region, sts_client_kwargs=dict( region_name=config.region, endpoint_url=f"https://sts.{config.region}.amazonaws.com", ), ) try: response = config_client.select_resource_config( Expression=query, Limit=100 ) for r in response.get("Results", []): resources.append(json.loads(r)) # Query Config for a specific account in all regions we care about while response.get("NextToken"): response = config_client.select_resource_config( Expression=query, Limit=100, NextToken=response["NextToken"] ) for r in response.get("Results", []): resources.append(json.loads(r)) except ClientError as e: log.error( { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "message": "Failed to query AWS Config", "query": query, "use_aggregator": use_aggregator, "account_id": account_id, "region": region, "error": str(e), }, exc_info=True, ) sentry_sdk.capture_exception() return resources
def dynamo_get_or_create_table(**dynamo_config): """ Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all data for Repokid. Set the global DYNAMO_TABLE object with a reference to the resource handle. Args: dynamo_config (kwargs): account_number (string) assume_role (string) optional session_name (string) region (string) endpoint (string) Returns: None """ global DYNAMO_TABLE if 'localhost' in dynamo_config['endpoint']: resource = boto3.resource('dynamodb', region_name='us-east-1', endpoint_url=dynamo_config['endpoint']) else: resource = boto3_cached_conn( 'dynamodb', service_type='resource', account_number=dynamo_config['account_number'], assume_role=dynamo_config.get('assume_role', None), session_name=dynamo_config['session_name'], region=dynamo_config['region']) try: table = resource.create_table( TableName='repokid_roles', KeySchema=[{ 'AttributeName': 'RoleId', 'KeyType': 'HASH' # Partition key }], AttributeDefinitions=[{ 'AttributeName': 'RoleId', 'AttributeType': 'S' }], ProvisionedThroughput={ 'ReadCapacityUnits': 5, 'WriteCapacityUnits': 5 }) table.meta.client.get_waiter('table_exists').wait( TableName='repokid_roles') # need a global secondary index to list all role IDs for a given account number table.update(AttributeDefinitions=[{ 'AttributeName': 'Account', 'AttributeType': 'S' }], GlobalSecondaryIndexUpdates=[{ 'Create': { 'IndexName': 'Account', 'KeySchema': [{ 'AttributeName': 'Account', 'KeyType': 'HASH' }], 'Projection': { 'NonKeyAttributes': ['RoleId'], 'ProjectionType': 'INCLUDE' }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 2, 'WriteCapacityUnits': 2 } } }]) except BotoClientError as e: if "ResourceInUseException" in e.message: table = resource.Table('repokid_roles') else: from repokid.repokid import LOGGER LOGGER.error(e) sys.exit(1) DYNAMO_TABLE = table
def dynamo_get_or_create_table(**dynamo_config): """ Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all data for Repokid. Return a table with a reference to the dynamo resource Args: dynamo_config (kwargs): account_number (string) assume_role (string) optional session_name (string) region (string) endpoint (string) Returns: dynamo_table object """ if 'localhost' in dynamo_config['endpoint']: resource = boto3.resource('dynamodb', region_name='us-east-1', endpoint_url=dynamo_config['endpoint']) else: resource = boto3_cached_conn( 'dynamodb', service_type='resource', account_number=dynamo_config['account_number'], assume_role=dynamo_config.get('assume_role', None), session_name=dynamo_config['session_name'], region=dynamo_config['region']) for table in resource.tables.all(): if table.name == 'repokid_roles': return table table = None try: table = resource.create_table( TableName='repokid_roles', KeySchema=[ { 'AttributeName': 'RoleId', 'KeyType': 'HASH' # Partition key } ], AttributeDefinitions=[ { 'AttributeName': 'RoleId', 'AttributeType': 'S' }, { 'AttributeName': 'RoleName', 'AttributeType': 'S' }, { 'AttributeName': 'Account', 'AttributeType': 'S' } ], ProvisionedThroughput={ 'ReadCapacityUnits': 50, 'WriteCapacityUnits': 50 }, GlobalSecondaryIndexes=[ { 'IndexName': 'Account', 'KeySchema': [ { 'AttributeName': 'Account', 'KeyType': 'HASH' } ], 'Projection': { 'ProjectionType': 'KEYS_ONLY', }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 } }, { 'IndexName': 'RoleName', 'KeySchema': [ { 'AttributeName': 'RoleName', 'KeyType': 'HASH' } ], 'Projection': { 'ProjectionType': 'KEYS_ONLY', }, 'ProvisionedThroughput': { 'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10 } }]) except BotoClientError as e: LOGGER.error(e) return table
def iam(sts, conn_dict): with mock_iam(): yield boto3_cached_conn("iam", **conn_dict)
def ec2(sts, conn_dict): with mock_ec2(): yield boto3_cached_conn("ec2", **conn_dict)
def dynamo_get_or_create_table( account_number: str, session_name: str, region: str, endpoint: str, assume_role: Optional[str] = "", ) -> Table: """ Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all data for Repokid. Return a table with a reference to the dynamo resource Args: account_number (string) assume_role (string) optional session_name (string) region (string) endpoint (string) Returns: dynamo_table object """ if "localhost" in endpoint: resource = boto3.resource("dynamodb", region_name="us-east-1", endpoint_url=endpoint) else: resource = boto3_cached_conn( "dynamodb", service_type="resource", account_number=account_number, assume_role=assume_role or None, session_name=session_name, region=region, ) for table in resource.tables.all(): if table.name == "repokid_roles": return table table = resource.create_table( TableName="repokid_roles", KeySchema=[{ "AttributeName": "RoleId", "KeyType": "HASH" }], # Partition key AttributeDefinitions=[ { "AttributeName": "RoleId", "AttributeType": "S" }, { "AttributeName": "RoleName", "AttributeType": "S" }, { "AttributeName": "Account", "AttributeType": "S" }, ], ProvisionedThroughput={ "ReadCapacityUnits": 50, "WriteCapacityUnits": 50 }, GlobalSecondaryIndexes=[ GlobalSecondaryIndexTypeDef({ "IndexName": "Account", "KeySchema": [{ "AttributeName": "Account", "KeyType": "HASH" }], "Projection": { "ProjectionType": "KEYS_ONLY" }, "ProvisionedThroughput": { "ReadCapacityUnits": 10, "WriteCapacityUnits": 10, }, }), GlobalSecondaryIndexTypeDef({ "IndexName": "RoleName", "KeySchema": [{ "AttributeName": "RoleName", "KeyType": "HASH" }], "Projection": { "ProjectionType": "KEYS_ONLY" }, "ProvisionedThroughput": { "ReadCapacityUnits": 10, "WriteCapacityUnits": 10, }, }), ], ) return table
def mock_iam_client(iam): yield boto3_cached_conn('iam', service_type='client', future_expiration_minutes=15, account_number='123456789012', region='us-east-1')
def dynamo_get_or_create_table(**dynamo_config): """ Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all data for Repokid. Return a table with a reference to the dynamo resource Args: dynamo_config (kwargs): account_number (string) assume_role (string) optional session_name (string) region (string) endpoint (string) Returns: dynamo_table object """ if "localhost" in dynamo_config["endpoint"]: resource = boto3.resource( "dynamodb", region_name="us-east-1", endpoint_url=dynamo_config["endpoint"] ) else: resource = boto3_cached_conn( "dynamodb", service_type="resource", account_number=dynamo_config["account_number"], assume_role=dynamo_config.get("assume_role", None), session_name=dynamo_config["session_name"], region=dynamo_config["region"], ) for table in resource.tables.all(): if table.name == "repokid_roles": return table table = None try: table = resource.create_table( TableName="repokid_roles", KeySchema=[{"AttributeName": "RoleId", "KeyType": "HASH"}], # Partition key AttributeDefinitions=[ {"AttributeName": "RoleId", "AttributeType": "S"}, {"AttributeName": "RoleName", "AttributeType": "S"}, {"AttributeName": "Account", "AttributeType": "S"}, ], ProvisionedThroughput={"ReadCapacityUnits": 50, "WriteCapacityUnits": 50}, GlobalSecondaryIndexes=[ { "IndexName": "Account", "KeySchema": [{"AttributeName": "Account", "KeyType": "HASH"}], "Projection": {"ProjectionType": "KEYS_ONLY"}, "ProvisionedThroughput": { "ReadCapacityUnits": 10, "WriteCapacityUnits": 10, }, }, { "IndexName": "RoleName", "KeySchema": [{"AttributeName": "RoleName", "KeyType": "HASH"}], "Projection": {"ProjectionType": "KEYS_ONLY"}, "ProvisionedThroughput": { "ReadCapacityUnits": 10, "WriteCapacityUnits": 10, }, }, ], ) except BotoClientError as e: LOGGER.error(e, exc_info=True) return table
async def update_role(event): log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "event": event, "message": "Working on event", } log.debug(log_data) if not isinstance(event, list): raise Exception("The passed event must be a list.") # Let's normalize all of the policies to JSON if they are not already for d in event: for i in d.get("inline_policies", []): if i.get("policy_document") and isinstance( i.get("policy_document"), dict): i["policy_document"] = json.dumps(i["policy_document"], escape_forward_slashes=False) if d.get("assume_role_policy_document", {}): if isinstance( d.get("assume_role_policy_document", {}).get("assume_role_policy_document"), dict, ): d["assume_role_policy_document"][ "assume_role_policy_document"] = json.dumps( d["assume_role_policy_document"] ["assume_role_policy_document"], escape_forward_slashes=False, ) bad_validation = RoleUpdaterRequest().validate(event, many=True) if bad_validation: log_data["error"] = bad_validation log.error(log_data) return { "error_msg": "invalid schema passed", "detail_error": bad_validation } event = RoleUpdaterRequest().load(event, many=True) result = {"success": False} for d in event: arn = d["arn"] aws_session_name = "roleupdater-" + d["requester"] account_number = await parse_account_id_from_arn(arn) role_name = await parse_role_name_from_arn(arn) # TODO: Make configurable client = boto3_cached_conn( "iam", account_number=account_number, assume_role=config.get("policies.role_name", "ConsoleMe"), session_name=aws_session_name, ) inline_policies = d.get("inline_policies", []) managed_policies = d.get("managed_policies", []) assume_role_doc = d.get("assume_role_policy_document", {}) tags = d.get("tags", []) if (not inline_policies and not managed_policies and not assume_role_doc and not tags): result[ "message"] = f"Invalid request. No response taken on event: {event}" return result try: for policy in inline_policies: await update_inline_policy(client, role_name, policy) for policy in managed_policies: await update_managed_policy(client, role_name, policy) if assume_role_doc: await update_assume_role_document(client, role_name, assume_role_doc) for tag in tags: await update_tags(client, role_name, tag) except ClientError as ce: result["message"] = ce.response["Error"] result["Traceback"] = traceback.format_exc() return result result["success"] = True return result
def detect_role_changes_and_update_cache(celery_app): """ This function detects role changes through event bridge rules, and forces a refresh of the roles. """ log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"} queue_arn = config.get( "event_bridge.detect_role_changes_and_update_cache.queue_arn", "" ).format(region=config.region) if not queue_arn: raise MissingConfigurationValue( "Unable to find required configuration value: " "`event_bridge.detect_role_changes_and_update_cache.queue_arn`" ) queue_name = queue_arn.split(":")[-1] queue_account_number = queue_arn.split(":")[4] queue_region = queue_arn.split(":")[3] # Optionally assume a role before receiving messages from the queue queue_assume_role = config.get( "event_bridge.detect_role_changes_and_update_cache.assume_role" ) sqs_client = boto3_cached_conn( "sqs", service_type="client", region=queue_region, retry_max_attempts=2, account_number=queue_account_number, assume_role=queue_assume_role, ) queue_url_res = sqs_client.get_queue_url(QueueName=queue_name) queue_url = queue_url_res.get("QueueUrl") if not queue_url: raise DataNotRetrievable(f"Unable to retrieve Queue URL for {queue_arn}") roles_to_update = set() messages = sqs_client.receive_message( QueueUrl=queue_url, MaxNumberOfMessages=10 ).get("Messages", []) while messages: processed_messages = [] for message in messages: try: message_body = json.loads(message["Body"]) decoded_message = json.loads(message_body["Message"]) role_name = decoded_message["detail"]["requestParameters"]["roleName"] role_account_id = decoded_message["account"] role_arn = f"arn:aws:iam::{role_account_id}:role/{role_name}" if role_arn not in roles_to_update: celery_app.send_task( "consoleme.celery_tasks.celery_tasks.refresh_iam_role", args=[role_arn], ) roles_to_update.add(role_arn) except Exception as e: log.error( {**log_data, "error": str(e), "raw_message": message}, exc_info=True ) sentry_sdk.capture_exception() processed_messages.append( { "Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"], } ) sqs_client.delete_message_batch(QueueUrl=queue_url, Entries=processed_messages) messages = sqs_client.receive_message( QueueUrl=queue_url, MaxNumberOfMessages=10 ).get("Messages", []) log.debug( { **log_data, "num_roles": len(roles_to_update), "message": "Triggered role cache update for roles that were created or changed", } ) return roles_to_update
def send_message(message_dict: Dict[str, Any], conn_details: Dict[str, Any]) -> None: client: SNSClient = boto3_cached_conn("sns", **conn_details) client.publish(TopicArn=CONFIG["dispatcher"]["from_rr_sns"], Message=json.dumps(message_dict))
def delete_message(receipt_handle: str, conn_details: Dict[str, Any]) -> None: client: SQSClient = boto3_cached_conn("sqs", **conn_details) client.delete_message(QueueUrl=CONFIG["dispatcher"]["to_rr_queue"], ReceiptHandle=receipt_handle)