def get_object(**kwargs):
client = kwargs.get("client")
assume_role = kwargs.get("assume_role")
if not client:
if assume_role:
client = boto3_cached_conn(
"s3",
account_number=kwargs.get("account_number"),
assume_role=assume_role,
session_name=kwargs.get("session_name", "ConsoleMe"),
region=kwargs.get("region", config.region),
retry_max_attempts=2,
client_kwargs=config.get("boto3.client_kwargs", {}),
)
else:
client = boto3.client("s3", **config.get("boto3.client_kwargs",
{}))
return client.get_object(Bucket=kwargs.get("Bucket"),
Key=kwargs.get("Key"))
def get(self, group=None):
user = DynamoHandler().get_requests_by_user("nag")
resource = boto3_cached_conn(
'dynamodb',
service_type='resource')
print (resource)
table = resource.Table("pet")
if not group:
response = table.scan()
#response = table.get_item(Key={"par": "test"})
self.write("Response {} ".format(json.dumps(response["Items"], indent=1)))
else:
print(group)
print("00000")
response = table.get_item(Key={"par": group})
print(response)
if response.get('Item', None):
print(dir(response))
self.write(" Response {} ".format(json.dumps(response["Item"],indent=1)))
else:
self.write("User :{} Not found".format(group))
def _get_arns(self):
"""
Gets a list of all Role ARNs in a given account, optionally limited by
class property ARN filter
:return: list of role ARNs
"""
client = boto3_cached_conn('iam',
service_type='client',
**self.conn_details)
account_arns = set()
for role in list_roles(**self.conn_details):
account_arns.add(role['Arn'])
for user in list_users(**self.conn_details):
account_arns.add(user['Arn'])
for page in client.get_paginator('list_policies').paginate(
Scope='Local'):
for policy in page['Policies']:
account_arns.add(policy['Arn'])
for page in client.get_paginator('list_groups').paginate():
for group in page['Groups']:
account_arns.add(group['Arn'])
result_arns = set()
for arn in self.arn_list:
if arn.lower() == 'all':
return account_arns
if arn not in account_arns:
self.current_app.logger.warn(
"Provided ARN {arn} not found in account.".format(arn=arn))
continue
result_arns.add(arn)
return list(result_arns)
def get_enabled_regions_for_account(account_id: str) -> Set[str]:
"""
Returns a list of regions enabled for an account based on an EC2 Describe Regions call. Can be overridden with a
global configuration of static regions (Configuration key: `celery.sync_regions`), or a configuration of specific
regions per account (Configuration key: `get_enabled_regions_for_account.{account_id}`)
"""
enabled_regions_for_account = config.get(
f"get_enabled_regions_for_account.{account_id}")
if enabled_regions_for_account:
return enabled_regions_for_account
celery_sync_regions = config.get("celery.sync_regions", [])
if celery_sync_regions:
return celery_sync_regions
client = boto3_cached_conn(
"ec2",
account_number=account_id,
assume_role=config.get("policies.role_name"),
read_only=True,
)
return set(r["RegionName"] for r in client.describe_regions()["Regions"])
def mock_classic_link(ec2):
conn = boto3_cached_conn("ec2",
service_type="client",
future_expiration_minutes=15,
account_number="012345678912",
region='us-east-1')
def describe_vpc_classic_link(**kwargs):
"""
# Too lazy to submit a PR to Moto -- not worth the trouble for ClassicLink
:param kwargs:
:return:
"""
return {
"Vpcs": [{
"ClassicLinkEnabled": True,
"Tags": [],
"VpcId": kwargs["VpcIds"][0]
}]
}
def describe_vpc_classic_link_dns_support(**kwargs):
"""
# Too lazy to submit a PR to Moto -- not worth the trouble for ClassicLink
:param kwargs:
:return:
"""
return {
"Vpcs": [{
"ClassicLinkDnsSupported": True,
"VpcId": kwargs["VpcIds"][0]
}]
}
setattr(conn, "describe_vpc_classic_link", describe_vpc_classic_link)
setattr(conn, "describe_vpc_classic_link_dns_support",
describe_vpc_classic_link_dns_support)
return conn
def apply_managed_policy_to_role(role: Dict, policy_name: str,
session_name: str) -> bool:
"""
Apply a managed policy to a role.
:param role: An AWS role dictionary (from a boto3 get_role or get_account_authorization_details call)
:param policy_name: Name of managed policy to add to role
:param session_name: Name of session to assume role with. This is an identifier that will be logged in CloudTrail
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
log_data = {
"function": function,
"role": role,
"policy_name": policy_name,
"session_name": session_name,
}
account_id = role.get("Arn").split(":")[4]
policy_arn = f"arn:aws:iam::{account_id}:policy/{policy_name}"
client = boto3_cached_conn(
"iam",
account_number=account_id,
assume_role=config.get("policies.role_name"),
session_name=session_name,
)
client.attach_role_policy(RoleName=role.get("RoleName"),
PolicyArn=policy_arn)
log_data["message"] = "Applied managed policy to role"
log.debug(log_data)
stats.count(
f"{function}.attach_role_policy",
tags={
"role": role.get("Arn"),
"policy": policy_arn
},
)
return True
def update_repoed_description(role_name: str, conn_details: Dict[str, Any]) -> None:
client: IAMClient = boto3_cached_conn("iam", **conn_details)
try:
description = client.get_role(RoleName=role_name)["Role"].get("Description", "")
except KeyError:
return
date_string = datetime.datetime.now(tz=datetime.timezone.utc).strftime("%m/%d/%y")
if "; Repokid repoed" in description:
new_description = re.sub(
r"; Repokid repoed [0-9]{2}\/[0-9]{2}\/[0-9]{2}",
f"; Repokid repoed {date_string}",
description,
)
else:
new_description = description + " ; Repokid repoed {}".format(date_string)
# IAM role descriptions have a max length of 1000, if our new length would be longer, skip this
if len(new_description) < 1000:
client.update_role_description(RoleName=role_name, Description=new_description)
else:
LOGGER.error(
"Unable to set repo description ({}) for role {}, length would be too long".format(
new_description, role_name
)
)
def query(
query: str, use_aggregator: bool = True, account_id: Optional[str] = None
) -> List:
resources = []
if use_aggregator:
config_client = boto3.client("config", region_name=config.region)
configuration_aggregator_name: str = config.get(
"aws_config.configuration_aggregator.name"
).format(region=config.region)
if not configuration_aggregator_name:
raise MissingConfigurationValue("Invalid configuration for aws_config")
response = config_client.select_aggregate_resource_config(
Expression=query,
ConfigurationAggregatorName=configuration_aggregator_name,
Limit=100,
)
for r in response.get("Results", []):
resources.append(json.loads(r))
while response.get("NextToken"):
response = config_client.select_aggregate_resource_config(
Expression=query,
ConfigurationAggregatorName=configuration_aggregator_name,
Limit=100,
NextToken=response["NextToken"],
)
for r in response.get("Results", []):
resources.append(json.loads(r))
return resources
else: # Don't use Config aggregator and instead query all the regions on an account
session = boto3.Session()
available_regions = session.get_available_regions("config")
excluded_regions = config.get(
"api_protect.exclude_regions",
["af-south-1", "ap-east-1", "ap-northeast-3", "eu-south-1", "me-south-1"],
)
regions = [x for x in available_regions if x not in excluded_regions]
for region in regions:
config_client = boto3_cached_conn(
"config",
account_number=account_id,
assume_role=config.get("policies.role_name"),
region=region,
sts_client_kwargs=dict(
region_name=config.region,
endpoint_url=f"https://sts.{config.region}.amazonaws.com",
),
)
try:
response = config_client.select_resource_config(
Expression=query, Limit=100
)
for r in response.get("Results", []):
resources.append(json.loads(r))
# Query Config for a specific account in all regions we care about
while response.get("NextToken"):
response = config_client.select_resource_config(
Expression=query, Limit=100, NextToken=response["NextToken"]
)
for r in response.get("Results", []):
resources.append(json.loads(r))
except ClientError as e:
log.error(
{
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Failed to query AWS Config",
"query": query,
"use_aggregator": use_aggregator,
"account_id": account_id,
"region": region,
"error": str(e),
},
exc_info=True,
)
sentry_sdk.capture_exception()
return resources
def dynamo_get_or_create_table(**dynamo_config):
"""
Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all
data for Repokid. Set the global DYNAMO_TABLE object with a reference to the resource handle.
Args:
dynamo_config (kwargs):
account_number (string)
assume_role (string) optional
session_name (string)
region (string)
endpoint (string)
Returns:
None
"""
global DYNAMO_TABLE
if 'localhost' in dynamo_config['endpoint']:
resource = boto3.resource('dynamodb',
region_name='us-east-1',
endpoint_url=dynamo_config['endpoint'])
else:
resource = boto3_cached_conn(
'dynamodb',
service_type='resource',
account_number=dynamo_config['account_number'],
assume_role=dynamo_config.get('assume_role', None),
session_name=dynamo_config['session_name'],
region=dynamo_config['region'])
try:
table = resource.create_table(
TableName='repokid_roles',
KeySchema=[{
'AttributeName': 'RoleId',
'KeyType': 'HASH' # Partition key
}],
AttributeDefinitions=[{
'AttributeName': 'RoleId',
'AttributeType': 'S'
}],
ProvisionedThroughput={
'ReadCapacityUnits': 5,
'WriteCapacityUnits': 5
})
table.meta.client.get_waiter('table_exists').wait(
TableName='repokid_roles')
# need a global secondary index to list all role IDs for a given account number
table.update(AttributeDefinitions=[{
'AttributeName': 'Account',
'AttributeType': 'S'
}],
GlobalSecondaryIndexUpdates=[{
'Create': {
'IndexName':
'Account',
'KeySchema': [{
'AttributeName': 'Account',
'KeyType': 'HASH'
}],
'Projection': {
'NonKeyAttributes': ['RoleId'],
'ProjectionType': 'INCLUDE'
},
'ProvisionedThroughput': {
'ReadCapacityUnits': 2,
'WriteCapacityUnits': 2
}
}
}])
except BotoClientError as e:
if "ResourceInUseException" in e.message:
table = resource.Table('repokid_roles')
else:
from repokid.repokid import LOGGER
LOGGER.error(e)
sys.exit(1)
DYNAMO_TABLE = table
def dynamo_get_or_create_table(**dynamo_config):
"""
Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all
data for Repokid. Return a table with a reference to the dynamo resource
Args:
dynamo_config (kwargs):
account_number (string)
assume_role (string) optional
session_name (string)
region (string)
endpoint (string)
Returns:
dynamo_table object
"""
if 'localhost' in dynamo_config['endpoint']:
resource = boto3.resource('dynamodb',
region_name='us-east-1',
endpoint_url=dynamo_config['endpoint'])
else:
resource = boto3_cached_conn(
'dynamodb',
service_type='resource',
account_number=dynamo_config['account_number'],
assume_role=dynamo_config.get('assume_role', None),
session_name=dynamo_config['session_name'],
region=dynamo_config['region'])
for table in resource.tables.all():
if table.name == 'repokid_roles':
return table
table = None
try:
table = resource.create_table(
TableName='repokid_roles',
KeySchema=[
{
'AttributeName': 'RoleId',
'KeyType': 'HASH' # Partition key
}
],
AttributeDefinitions=[
{
'AttributeName': 'RoleId',
'AttributeType': 'S'
},
{
'AttributeName': 'RoleName',
'AttributeType': 'S'
},
{
'AttributeName': 'Account',
'AttributeType': 'S'
}
],
ProvisionedThroughput={
'ReadCapacityUnits': 50,
'WriteCapacityUnits': 50
},
GlobalSecondaryIndexes=[
{
'IndexName': 'Account',
'KeySchema': [
{
'AttributeName': 'Account',
'KeyType': 'HASH'
}
],
'Projection': {
'ProjectionType': 'KEYS_ONLY',
},
'ProvisionedThroughput': {
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
}
},
{
'IndexName': 'RoleName',
'KeySchema': [
{
'AttributeName': 'RoleName',
'KeyType': 'HASH'
}
],
'Projection':
{
'ProjectionType': 'KEYS_ONLY',
},
'ProvisionedThroughput': {
'ReadCapacityUnits': 10,
'WriteCapacityUnits': 10
}
}])
except BotoClientError as e:
LOGGER.error(e)
return table
def iam(sts, conn_dict):
with mock_iam():
yield boto3_cached_conn("iam", **conn_dict)
def ec2(sts, conn_dict):
with mock_ec2():
yield boto3_cached_conn("ec2", **conn_dict)
def dynamo_get_or_create_table(
account_number: str,
session_name: str,
region: str,
endpoint: str,
assume_role: Optional[str] = "",
) -> Table:
"""
Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all
data for Repokid. Return a table with a reference to the dynamo resource
Args:
account_number (string)
assume_role (string) optional
session_name (string)
region (string)
endpoint (string)
Returns:
dynamo_table object
"""
if "localhost" in endpoint:
resource = boto3.resource("dynamodb",
region_name="us-east-1",
endpoint_url=endpoint)
else:
resource = boto3_cached_conn(
"dynamodb",
service_type="resource",
account_number=account_number,
assume_role=assume_role or None,
session_name=session_name,
region=region,
)
for table in resource.tables.all():
if table.name == "repokid_roles":
return table
table = resource.create_table(
TableName="repokid_roles",
KeySchema=[{
"AttributeName": "RoleId",
"KeyType": "HASH"
}], # Partition key
AttributeDefinitions=[
{
"AttributeName": "RoleId",
"AttributeType": "S"
},
{
"AttributeName": "RoleName",
"AttributeType": "S"
},
{
"AttributeName": "Account",
"AttributeType": "S"
},
],
ProvisionedThroughput={
"ReadCapacityUnits": 50,
"WriteCapacityUnits": 50
},
GlobalSecondaryIndexes=[
GlobalSecondaryIndexTypeDef({
"IndexName":
"Account",
"KeySchema": [{
"AttributeName": "Account",
"KeyType": "HASH"
}],
"Projection": {
"ProjectionType": "KEYS_ONLY"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 10,
"WriteCapacityUnits": 10,
},
}),
GlobalSecondaryIndexTypeDef({
"IndexName":
"RoleName",
"KeySchema": [{
"AttributeName": "RoleName",
"KeyType": "HASH"
}],
"Projection": {
"ProjectionType": "KEYS_ONLY"
},
"ProvisionedThroughput": {
"ReadCapacityUnits": 10,
"WriteCapacityUnits": 10,
},
}),
],
)
return table
def mock_iam_client(iam):
yield boto3_cached_conn('iam',
service_type='client',
future_expiration_minutes=15,
account_number='123456789012',
region='us-east-1')
def dynamo_get_or_create_table(**dynamo_config):
"""
Create a new table or get a reference to an existing Dynamo table named 'repokid_roles' that will store data all
data for Repokid. Return a table with a reference to the dynamo resource
Args:
dynamo_config (kwargs):
account_number (string)
assume_role (string) optional
session_name (string)
region (string)
endpoint (string)
Returns:
dynamo_table object
"""
if "localhost" in dynamo_config["endpoint"]:
resource = boto3.resource(
"dynamodb", region_name="us-east-1", endpoint_url=dynamo_config["endpoint"]
)
else:
resource = boto3_cached_conn(
"dynamodb",
service_type="resource",
account_number=dynamo_config["account_number"],
assume_role=dynamo_config.get("assume_role", None),
session_name=dynamo_config["session_name"],
region=dynamo_config["region"],
)
for table in resource.tables.all():
if table.name == "repokid_roles":
return table
table = None
try:
table = resource.create_table(
TableName="repokid_roles",
KeySchema=[{"AttributeName": "RoleId", "KeyType": "HASH"}], # Partition key
AttributeDefinitions=[
{"AttributeName": "RoleId", "AttributeType": "S"},
{"AttributeName": "RoleName", "AttributeType": "S"},
{"AttributeName": "Account", "AttributeType": "S"},
],
ProvisionedThroughput={"ReadCapacityUnits": 50, "WriteCapacityUnits": 50},
GlobalSecondaryIndexes=[
{
"IndexName": "Account",
"KeySchema": [{"AttributeName": "Account", "KeyType": "HASH"}],
"Projection": {"ProjectionType": "KEYS_ONLY"},
"ProvisionedThroughput": {
"ReadCapacityUnits": 10,
"WriteCapacityUnits": 10,
},
},
{
"IndexName": "RoleName",
"KeySchema": [{"AttributeName": "RoleName", "KeyType": "HASH"}],
"Projection": {"ProjectionType": "KEYS_ONLY"},
"ProvisionedThroughput": {
"ReadCapacityUnits": 10,
"WriteCapacityUnits": 10,
},
},
],
)
except BotoClientError as e:
LOGGER.error(e, exc_info=True)
return table
async def update_role(event):
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"event": event,
"message": "Working on event",
}
log.debug(log_data)
if not isinstance(event, list):
raise Exception("The passed event must be a list.")
# Let's normalize all of the policies to JSON if they are not already
for d in event:
for i in d.get("inline_policies", []):
if i.get("policy_document") and isinstance(
i.get("policy_document"), dict):
i["policy_document"] = json.dumps(i["policy_document"],
escape_forward_slashes=False)
if d.get("assume_role_policy_document", {}):
if isinstance(
d.get("assume_role_policy_document",
{}).get("assume_role_policy_document"),
dict,
):
d["assume_role_policy_document"][
"assume_role_policy_document"] = json.dumps(
d["assume_role_policy_document"]
["assume_role_policy_document"],
escape_forward_slashes=False,
)
bad_validation = RoleUpdaterRequest().validate(event, many=True)
if bad_validation:
log_data["error"] = bad_validation
log.error(log_data)
return {
"error_msg": "invalid schema passed",
"detail_error": bad_validation
}
event = RoleUpdaterRequest().load(event, many=True)
result = {"success": False}
for d in event:
arn = d["arn"]
aws_session_name = "roleupdater-" + d["requester"]
account_number = await parse_account_id_from_arn(arn)
role_name = await parse_role_name_from_arn(arn)
# TODO: Make configurable
client = boto3_cached_conn(
"iam",
account_number=account_number,
assume_role=config.get("policies.role_name", "ConsoleMe"),
session_name=aws_session_name,
)
inline_policies = d.get("inline_policies", [])
managed_policies = d.get("managed_policies", [])
assume_role_doc = d.get("assume_role_policy_document", {})
tags = d.get("tags", [])
if (not inline_policies and not managed_policies
and not assume_role_doc and not tags):
result[
"message"] = f"Invalid request. No response taken on event: {event}"
return result
try:
for policy in inline_policies:
await update_inline_policy(client, role_name, policy)
for policy in managed_policies:
await update_managed_policy(client, role_name, policy)
if assume_role_doc:
await update_assume_role_document(client, role_name,
assume_role_doc)
for tag in tags:
await update_tags(client, role_name, tag)
except ClientError as ce:
result["message"] = ce.response["Error"]
result["Traceback"] = traceback.format_exc()
return result
result["success"] = True
return result
def detect_role_changes_and_update_cache(celery_app):
"""
This function detects role changes through event bridge rules, and forces a refresh of the roles.
"""
log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"}
queue_arn = config.get(
"event_bridge.detect_role_changes_and_update_cache.queue_arn", ""
).format(region=config.region)
if not queue_arn:
raise MissingConfigurationValue(
"Unable to find required configuration value: "
"`event_bridge.detect_role_changes_and_update_cache.queue_arn`"
)
queue_name = queue_arn.split(":")[-1]
queue_account_number = queue_arn.split(":")[4]
queue_region = queue_arn.split(":")[3]
# Optionally assume a role before receiving messages from the queue
queue_assume_role = config.get(
"event_bridge.detect_role_changes_and_update_cache.assume_role"
)
sqs_client = boto3_cached_conn(
"sqs",
service_type="client",
region=queue_region,
retry_max_attempts=2,
account_number=queue_account_number,
assume_role=queue_assume_role,
)
queue_url_res = sqs_client.get_queue_url(QueueName=queue_name)
queue_url = queue_url_res.get("QueueUrl")
if not queue_url:
raise DataNotRetrievable(f"Unable to retrieve Queue URL for {queue_arn}")
roles_to_update = set()
messages = sqs_client.receive_message(
QueueUrl=queue_url, MaxNumberOfMessages=10
).get("Messages", [])
while messages:
processed_messages = []
for message in messages:
try:
message_body = json.loads(message["Body"])
decoded_message = json.loads(message_body["Message"])
role_name = decoded_message["detail"]["requestParameters"]["roleName"]
role_account_id = decoded_message["account"]
role_arn = f"arn:aws:iam::{role_account_id}:role/{role_name}"
if role_arn not in roles_to_update:
celery_app.send_task(
"consoleme.celery_tasks.celery_tasks.refresh_iam_role",
args=[role_arn],
)
roles_to_update.add(role_arn)
except Exception as e:
log.error(
{**log_data, "error": str(e), "raw_message": message}, exc_info=True
)
sentry_sdk.capture_exception()
processed_messages.append(
{
"Id": message["MessageId"],
"ReceiptHandle": message["ReceiptHandle"],
}
)
sqs_client.delete_message_batch(QueueUrl=queue_url, Entries=processed_messages)
messages = sqs_client.receive_message(
QueueUrl=queue_url, MaxNumberOfMessages=10
).get("Messages", [])
log.debug(
{
**log_data,
"num_roles": len(roles_to_update),
"message": "Triggered role cache update for roles that were created or changed",
}
)
return roles_to_update
def send_message(message_dict: Dict[str, Any],
conn_details: Dict[str, Any]) -> None:
client: SNSClient = boto3_cached_conn("sns", **conn_details)
client.publish(TopicArn=CONFIG["dispatcher"]["from_rr_sns"],
Message=json.dumps(message_dict))
def delete_message(receipt_handle: str, conn_details: Dict[str, Any]) -> None:
client: SQSClient = boto3_cached_conn("sqs", **conn_details)
client.delete_message(QueueUrl=CONFIG["dispatcher"]["to_rr_queue"],
ReceiptHandle=receipt_handle)
