def test_login_post_invalid_password(self):
from consoleme.lib.dynamo import UserDynamoHandler
ddb = UserDynamoHandler()
ddb.create_user("testuser", "correctpassword",
["group1", "*****@*****.**"])
body = json.dumps({
"username": "******",
"password": "******",
"after_redirect_uri": "/"
})
response = self.fetch("/api/v2/login", method="POST", body=body)
self.assertEqual(response.code, 403)
self.assertEqual(
json.loads(response.body),
{
"status":
"error",
"reason":
"authentication_failure",
"redirect_url":
None,
"status_code":
403,
"message":
None,
"errors": [
"User doesn't exist, or password is incorrect. ",
"Your next authentication failure will result in a 1 second wait. This wait time will expire after 60 seconds of no authentication failures.",
],
"data":
None,
},
)
async def api_add_user_to_group_or_raise(group_name, member_name, actor):
try:
group_info = await auth.get_group_info(group_name, members=False)
except Exception:
raise NoGroupsException("Unable to retrieve the specified group")
actor_groups = await auth.get_groups(actor)
can_add_remove_members = can_modify_members(actor, actor_groups, group_info)
if not can_add_remove_members:
raise UnauthorizedToAccess("Unauthorized to modify members of this group.")
try:
await add_user_to_group(member_name, group_name, actor)
except HttpError as e:
# Inconsistent GG API error - ignore failure for user already existing
if e.resp.reason == "duplicate":
pass
except UserAlreadyAMemberOfGroupException:
pass
except BulkAddPrevented:
dynamo_handler = UserDynamoHandler(actor)
dynamo_handler.add_request(
member_name,
group_name,
f"{actor} requesting on behalf of {member_name} from a bulk operation",
updated_by=actor,
)
return "REQUESTED"
return "ADDED"
def test_login_post_success(self):
from consoleme.lib.dynamo import UserDynamoHandler
ddb = UserDynamoHandler()
ddb.create_user("testuser2", "correctpassword",
["group1", "*****@*****.**"])
body = json.dumps({
"username": "******",
"password": "******",
"after_redirect_uri": "/",
})
response = self.fetch("/api/v2/login", method="POST", body=body)
self.assertEqual(response.code, 200)
self.assertEqual(
json.loads(response.body),
{
"status": "redirect",
"reason": "authenticated_redirect",
"redirect_url": "/",
"status_code": 200,
"message":
"User has successfully authenticated. Redirecting to their intended destination.",
"errors": None,
"data": None,
},
)
async def write_notification(notification: ConsoleMeUserNotification):
ddb = UserDynamoHandler()
await sync_to_async(ddb.notifications_table.put_item
)(Item=ddb._data_to_dynamo_replace(notification.dict())
)
await cache_notifications_to_redis_s3()
return True
def refresh_dynamic_config(ddb=None):
if not ddb:
# This function runs frequently. We provide the option to pass in a UserDynamoHandler
# so we don't need to import on every invocation
from consoleme.lib.dynamo import UserDynamoHandler
ddb = UserDynamoHandler()
return ddb.get_dynamic_config_dict()
def load_config_from_dynamo(self):
"""If enabled, we can load a configuration dynamically from Dynamo at a certain time interval. This reduces
the need for code redeploys to make configuration changes"""
from consoleme.lib.dynamo import UserDynamoHandler
from consoleme.lib.redis import RedisHandler
ddb = UserDynamoHandler()
red = RedisHandler().redis_sync()
while True:
dynamic_config = refresh_dynamic_config(ddb)
if dynamic_config and dynamic_config != self.config.get("dynamic_config"):
red.set(
"DYNAMIC_CONFIG_CACHE",
json.dumps(dynamic_config),
)
self.get_logger("config").debug(
{
"function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"message": "Dynamic configuration changes detected and loaded",
"dynamic_config": dynamic_config,
}
)
self.config["dynamic_config"] = dynamic_config
time.sleep(self.get("dynamic_config.dynamo_load_interval", 60))
async def get_app_pending_requests_policies(user):
dynamo_handler = UserDynamoHandler(user)
all_policy_requests = await dynamo_handler.get_all_policy_requests(
status="pending")
if not all_policy_requests:
all_policy_requests = []
return all_policy_requests
async def get_all_policy_requests(user, status=None):
dynamo_handler = UserDynamoHandler(user)
all_policy_requests = await dynamo_handler.get_all_policy_requests(
status=status)
if not all_policy_requests:
all_policy_requests = []
return all_policy_requests
async def get_all_pending_requests_api(user):
"""Get all pending requests and add the group's secondary approvers"""
dynamo_handler = UserDynamoHandler(user)
all_requests = await dynamo_handler.get_all_requests()
pending_requests = []
# Get secondary approvers for groups asynchronously, otherwise this can be a bottleneck
tasks = []
for req in all_requests:
if req.get("status") == "pending":
group = req.get("group")
task = asyncio.ensure_future(
auth.get_secondary_approvers(group, return_dict=True)
)
tasks.append(task)
pending_requests.append(req)
secondary_approver_responses = asyncio.gather(*tasks)
secondary_approver_mapping = {}
for mapping in await secondary_approver_responses:
for group, secondary_approvers in mapping.items():
secondary_approver_mapping[group] = ",".join(secondary_approvers)
for req in pending_requests:
req["secondary_approvers"] = secondary_approver_mapping.get(
req.get("group"), ""
)
return pending_requests
async def _get_extended_request(self, request_id, log_data):
dynamo = UserDynamoHandler(self.user)
requests = await dynamo.get_policy_requests(request_id=request_id)
if len(requests) == 0:
log_data["message"] = "Request with that ID not found"
log.warn(log_data)
stats.count(f"{log_data['function']}.not_found",
tags={"user": self.user})
raise NoMatchingRequest(log_data["message"])
if len(requests) > 1:
log_data["message"] = "Multiple requests with that ID found"
log.error(log_data)
stats.count(
f"{log_data['function']}.multiple_requests_found",
tags={"user": self.user},
)
raise InvalidRequestParameter(log_data["message"])
request = requests[0]
if request.get("version") != "2":
# Request format is not compatible with this endpoint version
raise InvalidRequestParameter(
"Request with that ID is not a v2 request")
extended_request = ExtendedRequestModel.parse_obj(
request.get("extended_request"))
return extended_request, request.get("last_updated")
async def fetch_notification(notification_id: str):
ddb = UserDynamoHandler()
notification = await sync_to_async(ddb.notifications_table.get_item
)(Key={
"predictable_id": notification_id
})
if notification.get("Item"):
return ConsoleMeUserNotification.parse_obj(notification["Item"])
async def add_user_to_group(
user_email: str,
google_group_email: str,
updated_by: Optional[str] = None,
role: str = "MEMBER",
dry_run: None = None,
service: Optional[Resource] = None,
request: Optional[Dict[str, Union[int, str]]] = None,
) -> Dict[str, bool]:
"""Add user member to group."""
dynamo = UserDynamoHandler()
result = {"done": True}
function = f"{__name__}.{sys._getframe().f_code.co_name}"
stats.count(function)
log_data = {
"function": function,
"user_email": user_email,
"google_group_email": google_group_email,
"updated_by": updated_by,
"role": role,
"dry_run": dry_run,
"message": "Adding user to group",
}
if not service:
service = await get_service("admin", "directory_v1",
google_group_email)
existing = await list_group_members(google_group_email, dry_run=dry_run)
if user_email in existing:
log_data[
"message"] = "Unable to add user to group. User is already a member."
log.warn(log_data)
result["done"] = False
result["message"] = log_data["message"]
raise UserAlreadyAMemberOfGroupException(result["message"])
group_info = await auth.get_group_info(google_group_email, members=False)
await raise_if_requires_bgcheck_and_no_bgcheck(user_email, group_info)
await raise_if_not_same_domain(user_email, group_info)
await raise_if_restricted(user_email, group_info)
await raise_if_bulk_add_disabled_and_no_request(group_info, request)
if not dry_run:
stats.count(
"google.add_user_to_group",
tags={
"user_email": user_email,
"google_group_email": google_group_email,
"updated_by": updated_by,
},
)
await insert_group_members_call(service, google_group_email,
user_email, role)
await dynamo.create_group_log_entry(google_group_email, user_email,
updated_by, "Added")
log.info(log_data)
return result
async def cache_notifications_to_redis_s3() -> Dict[str, int]:
function = f"{__name__}.{sys._getframe().f_code.co_name}"
current_time = int(time.time())
log_data = {"function": function}
ddb = UserDynamoHandler()
notifications_by_user_group = defaultdict(list)
all_notifications_l = await ddb.parallel_scan_table_async(
ddb.notifications_table)
changed_notifications = []
for existing_notification in all_notifications_l:
notification = ConsoleMeUserNotification.parse_obj(
existing_notification)
if current_time > notification.expiration:
notification.expired = True
changed_notifications.append(notification.dict())
for user_or_group in notification.users_or_groups:
notifications_by_user_group[user_or_group].append(
notification.dict())
if changed_notifications:
ddb.parallel_write_table(ddb.notifications_table,
changed_notifications)
if notifications_by_user_group:
for k, v in notifications_by_user_group.items():
notifications_by_user_group[k] = original_json.dumps(
v, cls=SetEncoder)
await store_json_results_in_redis_and_s3(
notifications_by_user_group,
redis_key=config.get("notifications.redis_key",
"ALL_NOTIFICATIONS"),
redis_data_type="hash",
s3_bucket=config.get("notifications.s3.bucket"),
s3_key=config.get("notifications.s3.key",
"notifications/all_notifications_v1.json.gz"),
)
log_data["num_user_groups_for_notifications"] = len(
notifications_by_user_group.keys())
log_data["num_notifications"] = len(all_notifications_l)
log.debug(log_data)
return {
"num_user_groups_to_notify": len(notifications_by_user_group.keys()),
"num_notifications": len(all_notifications_l),
}
async def get_existing_pending_request(user: str, group_info: Any) -> None:
dynamo_handler = UserDynamoHandler(user)
existing_requests = await sync_to_async(dynamo_handler.get_requests_by_user)(user)
if existing_requests:
for request in existing_requests:
if group_info.get("name") == request.get("group") and request.get(
"status"
) in ["pending"]:
return request
return None
async def migrate():
# Get all policy requests
# iterate through changes
# if has principal_arn, convert
dynamo = UserDynamoHandler("consoleme")
requests = await dynamo.get_all_policy_requests(status=None)
for request in requests:
changes = (request.get("extended_request",
{}).get("changes", {}).get("changes", []))
for change in changes:
if not change.get("principal_arn"):
continue
if not change.get("version") == "2.0":
continue
change["principal"] = {
"principal_arn": change["principal_arn"],
"principal_type": "AwsResource",
}
change.pop("principal_arn")
change["version"] = "3.0"
dynamo.parallel_write_table(dynamo.policy_requests_table, requests)
async def get_request_by_id(user, request_id):
"""Get request matching id and add the group's secondary approvers"""
dynamo_handler = UserDynamoHandler(user)
try:
requests = await sync_to_async(dynamo_handler.resolve_request_ids)([request_id])
for req in requests:
group = req.get("group")
secondary_approvers = await auth.get_secondary_approvers(group)
req["secondary_approvers"] = ",".join(secondary_approvers)
except NoMatchingRequest:
requests = []
return next(iter(requests), None)
async def remove_user_from_group(
user_email: str,
google_group_email: str,
updated_by: Optional[str] = None,
dry_run: None = None,
service: Optional[Resource] = None,
) -> Dict[str, bool]:
"""Remove user member to group."""
result = {"done": True}
dynamo = UserDynamoHandler()
function = f"{__name__}.{sys._getframe().f_code.co_name}"
stats.count(function)
log_data = {
"function": function,
"user_email": user_email,
"group": google_group_email,
"updated_by": updated_by,
"dry_run": dry_run,
"message": "Removing user from group",
}
log.debug(log_data)
group_info = await auth.get_group_info(google_group_email, members=False)
await raise_if_restricted(user_email, group_info)
if not service:
service = await get_service("admin", "directory_v1", google_group_email)
existing = await list_group_members(google_group_email, dry_run=dry_run)
if user_email in existing:
if not dry_run:
stats.count(
f"{function}.remove_user_from_group",
tags={
"user_email": user_email,
"google_group_email": google_group_email,
"updated_by": updated_by,
},
)
await delete_group_members_call(service, google_group_email, user_email)
await dynamo.create_group_log_entry(
google_group_email, user_email, updated_by, "Removed"
)
else:
log_data[
"message"
] = "Unable to remove user from group. User is not currently in the group."
log.warn(log_data)
result["done"] = False
result["message"] = log_data["message"]
raise NotAMemberException(result["message"])
return result
def load_config_from_dynamo_bg_thread(self):
"""If enabled, we can load a configuration dynamically from Dynamo at a certain time interval. This reduces
the need for code redeploys to make configuration changes"""
from consoleme.lib.dynamo import UserDynamoHandler
from consoleme.lib.redis import RedisHandler
ddb = UserDynamoHandler()
red = RedisHandler().redis_sync()
while threading.main_thread().is_alive():
self.load_config_from_dynamo(ddb, red)
# Wait till main exit flag is set OR a fixed timeout
if main_exit_flag.wait(timeout=self.get(
"dynamic_config.dynamo_load_interval", 60)):
break
class Policies:
"""
Policies internal plugin
"""
def __init__(self, ) -> None:
self.dynamo = UserDynamoHandler()
def error_count_by_arn(self):
try:
return self.dynamo.count_cloudtrail_errors_by_arn()
except Exception:
sentry_sdk.capture_exception()
return {}
async def get_errors_by_role(self, arn, n=5):
try:
return await self.dynamo.get_top_cloudtrail_errors_by_arn(arn, n)
except Exception:
sentry_sdk.capture_exception()
return {}
async def get_applications_associated_with_role(
self, arn: str) -> AppDetailsArray:
"""
This function returns applications associated with a role from configuration. You may want to override this
function to pull this information from an authoratative source.
:param arn: Role ARN
:return: AppDetailsArray
"""
apps_formatted = []
application_details = config.get("application_details", {})
for app, details in application_details.items():
apps_formatted.append(
AppDetailsModel(
name=app,
owner=details.get("owner"),
owner_url=details.get("owner_url"),
app_url=details.get("app_url"),
))
return AppDetailsArray(app_details=apps_formatted)
async def get_user_requests(user, groups):
"""Get requests relevant to a user.
A user sees requests they have made as well as requests where they are a
secondary approver
"""
dynamo_handler = UserDynamoHandler(user)
all_requests = await dynamo_handler.get_all_requests()
query = {
"domains":
config.get("dynamo.get_user_requests.domains", []),
"filters": [
{
"field": "extendedattributes.attributeName",
"values": ["secondary_approvers"],
"operator": "EQUALS",
},
{
"field": "extendedattributes.attributeValue",
"values": groups + [user],
"operator": "EQUALS",
},
],
"size":
500,
}
approver_groups = await auth.query_cached_groups(query=query)
approver_groups = [g["name"] for g in approver_groups]
requests = []
for req in all_requests:
if user == req.get("username", ""):
requests.append(req)
continue
group = req.get("group")
if group is None:
continue
if group in approver_groups + [user]:
requests.append(req)
return requests
def load_config_from_dynamo(self, ddb=None, red=None):
if not ddb:
from consoleme.lib.dynamo import UserDynamoHandler
ddb = UserDynamoHandler()
if not red:
from consoleme.lib.redis import RedisHandler
red = RedisHandler().redis_sync()
dynamic_config = refresh_dynamic_config(ddb)
if dynamic_config and dynamic_config != self.config.get("dynamic_config"):
red.set(
"DYNAMIC_CONFIG_CACHE",
json.dumps(dynamic_config),
)
self.get_logger("config").debug(
{
"function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"message": "Dynamic configuration changes detected and loaded",
}
)
self.config["dynamic_config"] = dynamic_config
async def detect_cloudtrail_denies_and_update_cache(
celery_app,
event_ttl=config.get(
"event_bridge.detect_cloudtrail_denies_and_update_cache.event_ttl",
86400),
max_num_messages_to_process=config.get(
"event_bridge.detect_cloudtrail_denies_and_update_cache.max_num_messages_to_process",
100,
),
) -> Dict[str, Any]:
log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"}
dynamo = UserDynamoHandler()
queue_arn = config.get(
"event_bridge.detect_cloudtrail_denies_and_update_cache.queue_arn",
"").format(region=config.region)
if not queue_arn:
raise MissingConfigurationValue(
"Unable to find required configuration value: "
"`event_bridge.detect_cloudtrail_denies_and_update_cache.queue_arn`"
)
queue_name = queue_arn.split(":")[-1]
queue_account_number = queue_arn.split(":")[4]
queue_region = queue_arn.split(":")[3]
# Optionally assume a role before receiving messages from the queue
queue_assume_role = config.get(
"event_bridge.detect_cloudtrail_denies_and_update_cache.assume_role")
# Modify existing cloudtrail deny samples
all_cloudtrail_denies_l = await dynamo.parallel_scan_table_async(
dynamo.cloudtrail_table)
all_cloudtrail_denies = {}
for cloudtrail_deny in all_cloudtrail_denies_l:
all_cloudtrail_denies[cloudtrail_deny["request_id"]] = cloudtrail_deny
sqs_client = await sync_to_async(boto3_cached_conn)(
"sqs",
service_type="client",
region=queue_region,
retry_max_attempts=2,
account_number=queue_account_number,
assume_role=queue_assume_role,
client_kwargs=config.get("boto3.client_kwargs", {}),
)
queue_url_res = await sync_to_async(sqs_client.get_queue_url
)(QueueName=queue_name)
queue_url = queue_url_res.get("QueueUrl")
if not queue_url:
raise DataNotRetrievable(
f"Unable to retrieve Queue URL for {queue_arn}")
messages_awaitable = await sync_to_async(sqs_client.receive_message
)(QueueUrl=queue_url,
MaxNumberOfMessages=10)
new_events = 0
messages = messages_awaitable.get("Messages", [])
num_events = 0
reached_limit_on_num_messages_to_process = False
while messages:
if num_events >= max_num_messages_to_process:
reached_limit_on_num_messages_to_process = True
break
processed_messages = []
for message in messages:
try:
message_body = json.loads(message["Body"])
try:
if "Message" in message_body:
decoded_message = json.loads(
message_body["Message"])["detail"]
else:
decoded_message = message_body["detail"]
except Exception as e:
log.error({
**log_data,
"message": "Unable to process Cloudtrail message",
"message_body": message_body,
"error": str(e),
})
sentry_sdk.capture_exception()
continue
event_name = decoded_message.get("eventName")
event_source = decoded_message.get("eventSource")
for event_source_substitution in config.get(
"event_bridge.detect_cloudtrail_denies_and_update_cache.event_bridge_substitutions",
[".amazonaws.com"],
):
event_source = event_source.replace(
event_source_substitution, "")
event_time = decoded_message.get("eventTime")
utc_time = datetime.strptime(event_time, "%Y-%m-%dT%H:%M:%SZ")
epoch_event_time = int(
(utc_time - datetime(1970, 1, 1)).total_seconds())
# Skip entries older than a day
if int(time.time()) - 86400 > epoch_event_time:
continue
try:
session_name = decoded_message["userIdentity"][
"arn"].split("/")[-1]
except (
IndexError,
KeyError,
): # If IAM user, there won't be a session name
session_name = ""
try:
principal_arn = decoded_message["userIdentity"][
"sessionContext"]["sessionIssuer"]["arn"]
except KeyError: # Skip events without a parsable ARN
continue
event_call = f"{event_source}:{event_name}"
ct_event = dict(
error_code=decoded_message.get("errorCode"),
error_message=decoded_message.get("errorMessage"),
arn=principal_arn,
# principal_owner=owner,
session_name=session_name,
source_ip=decoded_message["sourceIPAddress"],
event_call=event_call,
epoch_event_time=epoch_event_time,
ttl=epoch_event_time + event_ttl,
count=1,
)
resource = await get_resource_from_cloudtrail_deny(
ct_event, decoded_message)
ct_event["resource"] = resource
request_id = f"{principal_arn}-{session_name}-{event_call}-{resource}"
ct_event["request_id"] = request_id
generated_policy = await generate_policy_from_cloudtrail_deny(
ct_event)
if generated_policy:
ct_event["generated_policy"] = generated_policy
if all_cloudtrail_denies.get(request_id):
existing_count = all_cloudtrail_denies[request_id].get(
"count", 1)
ct_event["count"] += existing_count
all_cloudtrail_denies[request_id] = ct_event
else:
all_cloudtrail_denies[request_id] = ct_event
new_events += 1
num_events += 1
except Exception as e:
log.error({**log_data, "error": str(e)}, exc_info=True)
sentry_sdk.capture_exception()
processed_messages.append({
"Id":
message["MessageId"],
"ReceiptHandle":
message["ReceiptHandle"],
})
if processed_messages:
await sync_to_async(sqs_client.delete_message_batch
)(QueueUrl=queue_url,
Entries=processed_messages)
await sync_to_async(dynamo.batch_write_cloudtrail_events
)(all_cloudtrail_denies.values())
messages_awaitable = await sync_to_async(sqs_client.receive_message
)(QueueUrl=queue_url,
MaxNumberOfMessages=10)
messages = messages_awaitable.get("Messages", [])
if reached_limit_on_num_messages_to_process:
# We hit our limit. Let's spawn another task immediately to process remaining messages
celery_app.send_task(
"consoleme.celery_tasks.celery_tasks.cache_cloudtrail_denies", )
log_data["message"] = "Successfully cached Cloudtrail Access Denies"
log_data["num_events"] = num_events
log_data["new_events"] = new_events
log.debug(log_data)
return log_data
class UserManagementHandler(BaseAPIV2Handler):
"""
Handles creating and updating users. Only authorized users are allowed to access this endpoint.
"""
def initialize(self):
self.ddb = UserDynamoHandler()
async def post(self):
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"user": self.user,
"message": "Create/Update User",
"user-agent": self.request.headers.get("User-Agent"),
"request_id": self.request_uuid,
"ip": self.ip,
}
generic_error_message = "Unable to create/update user"
log.debug(log_data)
# Checks authz levels of current user
if not can_admin_all(self.user, self.groups):
errors = ["User is not authorized to access this endpoint."]
await handle_generic_error_response(self, generic_error_message,
errors, 403, "unauthorized",
log_data)
return
request = UserManagementModel.parse_raw(self.request.body)
log_data["requested_user"] = request.username
if request.user_management_action.value == "create":
log.debug({
**log_data,
"message": "Creating user",
"requested_user": request.username,
"requested_groups": request.groups,
})
# Fails if password is not strong enough.
password_strength_errors = await check_password_strength(
request.password)
if password_strength_errors:
await handle_generic_error_response(
self,
password_strength_errors["message"],
password_strength_errors["errors"],
403,
"weak_password",
log_data,
)
return
self.ddb.create_user(
request.username,
request.password,
request.groups,
)
res = WebResponse(
status="success",
status_code=200,
message=f"Successfully created user {request.username}.",
)
self.write(res.json())
return
elif request.user_management_action.value == "update":
log.debug({
**log_data,
"message": "Updating user",
"requested_user": request.username,
"requested_groups": request.groups,
})
if request.password:
# Fails if password is not strong enough.
password_strength_errors = await check_password_strength(
request.password)
if password_strength_errors:
await handle_generic_error_response(
self,
password_strength_errors["message"],
password_strength_errors["errors"],
403,
"weak_password",
log_data,
)
return
self.ddb.update_user(
request.username,
request.password,
request.groups,
)
res = WebResponse(
status="success",
status_code=200,
message=f"Successfully updated user {request.username}.",
)
self.write(res.json())
return
elif request.user_management_action.value == "delete":
log.debug({
**log_data,
"message": "Deleting user",
"requested_user": request.username,
})
self.ddb.delete_user(request.username, )
res = WebResponse(
status="success",
status_code=200,
message=f"Successfully deleted user {request.username}.",
)
self.write(res.json())
return
else:
errors = ["Change type is not supported by this endpoint."]
await handle_generic_error_response(self, generic_error_message,
errors, 403, "invalid_request",
log_data)
return
from collections import defaultdict
import simplejson as json
from boto3.dynamodb.types import Binary # noqa
from consoleme.config import config
from consoleme.lib.aws import get_iam_principal_owner, simulate_iam_principal_action
from consoleme.lib.cache import store_json_results_in_redis_and_s3
from consoleme.lib.dynamo import UserDynamoHandler
from consoleme.lib.json_encoder import SetEncoder
from consoleme.lib.notifications.models import (
ConsoleMeUserNotification,
ConsoleMeUserNotificationAction,
)
ddb = UserDynamoHandler()
class CloudTrail:
async def process_cloudtrail_errors(
self,
aws,
notification_ttl_seconds=config.get(
"process_cloudtrail_errors.notification_ttl", 86400),
) -> object:
"""
Processes Cloudtrail Errors that were cached by the `cache_cloudtrail_denies` celery task. Generates and returns
count data. If configured, generates notifications to end-users based on policies that can be generated
:param notification_ttl_seconds:
:return:
def test_create_user(self):
from consoleme.config import config
headers = {
config.get("auth.user_header_name"):
"*****@*****.**",
config.get("auth.groups_header_name"): "groupa,groupb,groupc",
}
body = json.dumps({
"user_management_action": "create",
"username": "******",
"password": "******",
"groups": ["group1", "group2", "group3"],
})
response = self.fetch("/api/v2/user",
headers=headers,
method="POST",
body=body)
self.assertEqual(response.code, 200)
self.assertEqual(
json.loads(response.body),
{
"status": "success",
"reason": None,
"redirect_url": None,
"status_code": 200,
"message": "Successfully created user testuser3.",
"errors": None,
"data": None,
},
)
# Verify new user works
from consoleme.lib.dynamo import UserDynamoHandler
from consoleme.models import LoginAttemptModel
ddb = UserDynamoHandler()
login_attempt_success = LoginAttemptModel(username="******",
password="******")
should_pass = async_to_sync(
ddb.authenticate_user)(login_attempt_success)
self.assertEqual(
should_pass.dict(),
{
"authenticated": True,
"errors": None,
"username": "******",
"groups": ["group1", "group2", "group3"],
},
)
login_attempt_fail = LoginAttemptModel(username="******",
password="******")
should_fail = async_to_sync(ddb.authenticate_user)(login_attempt_fail)
self.assertEqual(
should_fail.dict(),
{
"authenticated":
False,
"errors": [
"User doesn't exist, or password is incorrect. ",
"Your next authentication failure will result in a 1 second wait. "
"This wait time will expire after 60 seconds of no authentication failures.",
],
"username":
None,
"groups":
None,
},
)
# Update password
body = json.dumps({
"user_management_action": "update",
"username": "******",
"password": "******",
})
response = self.fetch("/api/v2/user",
headers=headers,
method="POST",
body=body)
self.assertEqual(response.code, 200)
self.assertEqual(
json.loads(response.body),
{
"status": "success",
"reason": None,
"redirect_url": None,
"status_code": 200,
"message": "Successfully updated user testuser3.",
"errors": None,
"data": None,
},
)
# Update groups
body = json.dumps({
"user_management_action": "update",
"username": "******",
"groups": ["group1", "group2", "group3", "newgroup"],
})
response = self.fetch("/api/v2/user",
headers=headers,
method="POST",
body=body)
self.assertEqual(response.code, 200)
self.assertEqual(
json.loads(response.body),
{
"status": "success",
"reason": None,
"redirect_url": None,
"status_code": 200,
"message": "Successfully updated user testuser3.",
"errors": None,
"data": None,
},
)
# Update groups and password AT THE SAME TIME!!1
body = json.dumps({
"user_management_action":
"update",
"username":
"******",
"password":
"******",
"groups": ["group1", "group2", "group3", "newgroup", "newgroup2"],
})
response = self.fetch("/api/v2/user",
headers=headers,
method="POST",
body=body)
self.assertEqual(response.code, 200)
self.assertEqual(
json.loads(response.body),
{
"status": "success",
"reason": None,
"redirect_url": None,
"status_code": 200,
"message": "Successfully updated user testuser3.",
"errors": None,
"data": None,
},
)
# Delete the user
body = json.dumps({
"user_management_action": "delete",
"username": "******",
})
response = self.fetch("/api/v2/user",
headers=headers,
method="POST",
body=body)
self.assertEqual(response.code, 200)
self.assertEqual(
json.loads(response.body),
{
"status": "success",
"reason": None,
"redirect_url": None,
"status_code": 200,
"message": "Successfully deleted user testuser3.",
"errors": None,
"data": None,
},
)
def __init__(self, ) -> None:
self.dynamo = UserDynamoHandler()
class UserRegistrationHandler(tornado.web.RequestHandler):
"""
Allows user registration if it is configured.
"""
def initialize(self):
self.ddb = UserDynamoHandler()
async def post(self):
# TODO: Send verification e-mail to proposed user
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"message": "Attempting to register user",
"user-agent": self.request.headers.get("User-Agent"),
}
generic_error_message: str = "User registration failed"
# Fail if getting users by password is not enabled
if not config.get("auth.get_user_by_password"):
errors = [
"Expected configuration `auth.get_user_by_password`, but it is not enabled."
]
await handle_generic_error_response(self, generic_error_message,
errors, 403, "not_configured",
log_data)
return
# Fail if user registration not allowed
if not config.get("auth.allow_user_registration"):
errors = [
"Expected configuration `auth.allow_user_registration`, but it is not enabled."
]
await handle_generic_error_response(self, generic_error_message,
errors, 403, "not_configured",
log_data)
return
registration_attempt = RegistrationAttemptModel.parse_raw(
self.request.body)
log_data["username"] = registration_attempt.username
# Fail if username not valid email address
try:
if not validate_email(registration_attempt.username):
errors = ["Username must be a valid e-mail address."]
await handle_generic_error_response(
self,
generic_error_message,
errors,
403,
"invalid_request",
log_data,
)
return
except Exception as e:
sentry_sdk.capture_exception()
await handle_generic_error_response(self, generic_error_message,
[str(e)], 403,
"invalid_request", log_data)
return
# Fail if user already exists
if await self.ddb.get_user(registration_attempt.username):
errors = ["User already exists"]
await handle_generic_error_response(self, generic_error_message,
errors, 403, "invalid_request",
log_data)
return
# Fails if password is not strong enough.
password_strength_errors = await check_password_strength(
registration_attempt.password)
if password_strength_errors:
await handle_generic_error_response(
self,
password_strength_errors["message"],
password_strength_errors["errors"],
403,
"weak_password",
log_data,
)
return
self.ddb.create_user(registration_attempt.username,
registration_attempt.password)
res = WebResponse(
status="success",
status_code=200,
message=
f"Successfully created user {registration_attempt.username}.",
)
self.write(res.json())
def initialize(self):
self.ddb = UserDynamoHandler()
async def get(self, request_id):
"""
GET /api/v2/requests/{request_id}
"""
tags = {"user": self.user}
stats.count("RequestDetailHandler.get", tags=tags)
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"user": self.user,
"message": "Get request details",
"user-agent": self.request.headers.get("User-Agent"),
"request_id": self.request_uuid,
"policy_request_id": request_id,
}
log.debug(log_data)
if config.get("policy_editor.disallow_contractors",
True) and self.contractor:
if self.user not in config.get(
"groups.can_bypass_contractor_restrictions", []):
self.write_error(
403, message="Only FTEs are authorized to view this page.")
return
try:
extended_request, last_updated = await self._get_extended_request(
request_id, log_data)
except InvalidRequestParameter as e:
sentry_sdk.capture_exception(tags={"user": self.user})
self.write_error(400, message="Error validating input: " + str(e))
return
except NoMatchingRequest as e:
sentry_sdk.capture_exception(tags={"user": self.user})
self.write_error(404, message="Error getting request:" + str(e))
return
# Run these tasks concurrently.
concurrent_results = await asyncio.gather(
populate_old_policies(extended_request, self.user),
populate_cross_account_resource_policies(extended_request,
self.user),
)
extended_request = concurrent_results[0]
populate_cross_account_resource_policies_result = concurrent_results[1]
if populate_cross_account_resource_policies_result["changed"]:
extended_request = populate_cross_account_resource_policies_result[
"extended_request"]
# Update in dynamo with the latest resource policy changes
dynamo = UserDynamoHandler(self.user)
updated_request = await dynamo.write_policy_request_v2(
extended_request)
last_updated = updated_request.get("last_updated")
can_approve_reject = (can_admin_policies(self.user, self.groups), )
can_update_cancel = await can_update_cancel_requests_v2(
extended_request.requester_email, self.user, self.groups)
can_move_back_to_pending = await can_move_back_to_pending_v2(
extended_request, last_updated, self.user, self.groups)
# In the future request_specific_config will have specific approvers for specific changes based on ABAC
request_specific_config = {
"can_approve_reject": can_approve_reject,
"can_update_cancel": can_update_cancel,
"can_move_back_to_pending": can_move_back_to_pending,
}
template = None
# Force a refresh of the role in Redis/DDB
arn_parsed = parse_arn(extended_request.arn)
if arn_parsed["service"] == "iam" and arn_parsed["resource"] == "role":
iam_role = await aws.fetch_iam_role(arn_parsed["account"],
extended_request.arn)
template = iam_role.get("templated")
response = {
"request": extended_request.json(),
"last_updated": last_updated,
"request_config": request_specific_config,
"template": template,
}
self.write(response)
async def post(self):
"""
POST /api/v2/request
Request example JSON: (Request Schema is RequestCreationModel in models.py)
{
"changes": {
"changes": [
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "inline_policy",
"action": "attach",
"policy": {
"policy_document": {
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:ListMultipartUploadParts*",
"s3:ListBucket"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::curtis-nflx-test/*",
"arn:aws:s3:::curtis-nflx-test"
],
"Sid": "cmccastrapel159494014dsd1shak"
},
{
"Action": [
"ec2:describevolumes",
"ec2:detachvolume",
"ec2:describelicenses",
"ec2:AssignIpv6Addresses",
"ec2:reportinstancestatus"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Sid": "cmccastrapel1594940141hlvvv"
},
{
"Action": [
"sts:AssumeRole"
],
"Effect": "Allow",
"Resource": [
"arn:aws:iam::123456789012:role/curtisTestInstanceProfile"
],
"Sid": "cmccastrapel1596483596easdits"
}
]
}
}
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "assume_role_policy",
"policy": {
"policy_document": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::123456789012:role/consolemeInstanceProfile"
},
"Sid": "AllowConsoleMeProdAssumeRolses"
}
],
"Version": "2012-10-17"
}
}
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "managed_policy",
"policy_name": "ApiProtect",
"action": "attach",
"arn": "arn:aws:iam::123456789012:policy/ApiProtect"
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "managed_policy",
"policy_name": "TagProtect",
"action": "detach",
"arn": "arn:aws:iam::123456789012:policy/TagProtect"
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "inline_policy",
"policy_name": "random_policy254",
"action": "attach",
"policy": {
"policy_document": {
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"ec2:AssignIpv6Addresses"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Sid": "cmccastrapel1594940141shakabcd"
}
]
}
}
}
]
},
"justification": "testing this out.",
"admin_auto_approve": false
}
Response example JSON: (Response Schema is RequestCreationResponse in models.py)
{
"errors": 1,
"request_created": true,
"request_id": "0c9fb298-c8ea-4d50-917c-3212da07b3ad",
"action_results": [
{
"status": "success",
"message": "Success description"
},
{
"status": "error",
"message": "Error description"
}
]
}
"""
if config.get("policy_editor.disallow_contractors",
True) and self.contractor:
if self.user not in config.get(
"groups.can_bypass_contractor_restrictions", []):
raise MustBeFte("Only FTEs are authorized to view this page.")
tags = {"user": self.user}
stats.count("RequestHandler.post", tags=tags)
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"user": self.user,
"message": "Create request initialization",
"user-agent": self.request.headers.get("User-Agent"),
"request_id": self.request_uuid,
"ip": self.ip,
"admin_auto_approved": False,
"probe_auto_approved": False,
}
log.debug(log_data)
try:
# Validate the model
changes = RequestCreationModel.parse_raw(self.request.body)
extended_request = await generate_request_from_change_model_array(
changes, self.user)
log_data["request"] = extended_request.dict()
log.debug(log_data)
admin_approved = False
approval_probe_approved = False
# TODO: Provide a note to the requester that admin_auto_approve will apply the requested policies only.
# It will not automatically apply generated policies. The administrative user will need to visit the policy
# Request page to do this manually.
if changes.admin_auto_approve:
# make sure user is allowed to use admin_auto_approve
can_manage_policy_request = (can_admin_policies(
self.user, self.groups), )
if can_manage_policy_request:
extended_request.request_status = RequestStatus.approved
admin_approved = True
extended_request.reviewer = self.user
self_approval_comment = CommentModel(
id=str(uuid.uuid4()),
timestamp=int(time.time()),
user_email=self.user,
user=extended_request.requester_info,
last_modified=int(time.time()),
text=f"Self-approved by admin: {self.user}",
)
extended_request.comments.append(self_approval_comment)
log_data["admin_auto_approved"] = True
log_data["request"] = extended_request.dict()
log.debug(log_data)
stats.count(
f"{log_data['function']}.post.admin_auto_approved",
tags={"user": self.user},
)
else:
# someone is trying to use admin bypass without being an admin, don't allow request to proceed
stats.count(
f"{log_data['function']}.post.unauthorized_admin_bypass",
tags={"user": self.user},
)
log_data[
"message"] = "Unauthorized user trying to use admin bypass"
log.error(log_data)
await write_json_error("Unauthorized", obj=self)
return
else:
# If admin auto approve is false, check for auto-approve probe eligibility
is_eligible_for_auto_approve_probe = (
await is_request_eligible_for_auto_approval(
extended_request, self.user))
# If we have only made requests that are eligible for auto-approval probe, check against them
if is_eligible_for_auto_approve_probe:
should_auto_approve_request = await should_auto_approve_policy_v2(
extended_request, self.user, self.groups)
if should_auto_approve_request["approved"]:
extended_request.request_status = RequestStatus.approved
approval_probe_approved = True
stats.count(
f"{log_data['function']}.probe_auto_approved",
tags={"user": self.user},
)
approving_probes = []
for approving_probe in should_auto_approve_request[
"approving_probes"]:
approving_probe_comment = CommentModel(
id=str(uuid.uuid4()),
timestamp=int(time.time()),
user_email=
f"Auto-Approve Probe: {approving_probe['name']}",
last_modified=int(time.time()),
text=
f"Policy {approving_probe['policy']} auto-approved by probe: {approving_probe['name']}",
)
extended_request.comments.append(
approving_probe_comment)
approving_probes.append(approving_probe["name"])
extended_request.reviewer = (
f"Auto-Approve Probe: {','.join(approving_probes)}"
)
log_data["probe_auto_approved"] = True
log_data["request"] = extended_request.dict()
log.debug(log_data)
dynamo = UserDynamoHandler(self.user)
request = await dynamo.write_policy_request_v2(extended_request)
log_data["message"] = "New request created in Dynamo"
log_data["request"] = extended_request.dict()
log_data["dynamo_request"] = request
log.debug(log_data)
except (InvalidRequestParameter, ValidationError) as e:
log_data["message"] = "Validation Exception"
log.error(log_data, exc_info=True)
stats.count(f"{log_data['function']}.validation_exception",
tags={"user": self.user})
self.write_error(400, message="Error validating input: " + str(e))
if config.get("development"):
raise
return
except Exception as e:
log_data[
"message"] = "Unknown Exception occurred while parsing request"
log.error(log_data, exc_info=True)
stats.count(f"{log_data['function']}.exception",
tags={"user": self.user})
sentry_sdk.capture_exception(tags={"user": self.user})
self.write_error(500, message="Error parsing request: " + str(e))
if config.get("development"):
raise
return
# If here, request has been successfully created
response = RequestCreationResponse(
errors=0,
request_created=True,
request_id=extended_request.id,
request_url=f"/policies/request/{extended_request.id}",
action_results=[],
)
# If approved is true due to an auto-approval probe or admin auto-approval, apply the non-autogenerated changes
if extended_request.request_status == RequestStatus.approved:
for change in extended_request.changes.changes:
if change.autogenerated:
continue
policy_request_modification_model = (
PolicyRequestModificationRequestModel.parse_obj({
"modification_model": {
"command": "apply_change",
"change_id": change.id,
}
}))
policy_apply_response = (
await parse_and_apply_policy_request_modification(
extended_request,
policy_request_modification_model,
self.user,
self.groups,
int(time.time()),
approval_probe_approved,
))
response.errors = policy_apply_response.errors
response.action_results = policy_apply_response.action_results
# Update in dynamo
await dynamo.write_policy_request_v2(extended_request)
account_id = await get_resource_account(extended_request.arn)
# Force a refresh of the role in Redis/DDB
arn_parsed = parse_arn(extended_request.arn)
if arn_parsed["service"] == "iam" and arn_parsed[
"resource"] == "role":
await aws.fetch_iam_role(account_id,
extended_request.arn,
force_refresh=True)
log_data["request"] = extended_request.dict()
log_data["message"] = "Applied changes based on approved request"
log_data["response"] = response.dict()
log.debug(log_data)
await aws.send_communications_new_policy_request(
extended_request, admin_approved, approval_probe_approved)
self.write(response.json())
await self.finish()
await cache_all_policy_requests()
return