def write(self, chunk: Union[str, bytes, dict]) -> None:
if config.get("_security_risk_full_debugging.enabled"):
self.responses.append(chunk)
super(BaseMtlsHandler, self).write(chunk)
async def get_user(self, headers: dict = None):
"""Get the user identity."""
if config.get("auth.get_user_by_header"):
return await self.get_user_by_header(headers)
else:
raise Exception("auth.get_user not configured")
from consoleme.config import config
from consoleme.exceptions.exceptions import (
BackgroundCheckNotPassedException,
MissingConfigurationValue,
)
from consoleme.lib.cache import retrieve_json_data_from_redis_or_s3
from consoleme.lib.plugins import get_plugin_by_name
from consoleme.lib.redis import RedisHandler, redis_hget
from consoleme.models import CloneRoleRequestModel, RoleCreationRequestModel
ALL_IAM_MANAGED_POLICIES: dict = {}
ALL_IAM_MANAGED_POLICIES_LAST_UPDATE: int = 0
log = config.get_logger(__name__)
auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
@rate_limited()
def create_managed_policy(cloudaux, name, path, policy, description):
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"cloudaux": cloudaux,
"name": name,
"path": path,
"policy": policy,
"description": "description",
"message": "Creating Managed Policy",
}
log.debug(log_data)
async def post(self):
"""
POST /api/v2/request
Request example JSON: (Request Schema is RequestCreationModel in models.py)
{
"changes": {
"changes": [
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "inline_policy",
"action": "attach",
"policy": {
"policy_document": {
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"s3:ListMultipartUploadParts*",
"s3:ListBucket"
],
"Effect": "Allow",
"Resource": [
"arn:aws:s3:::curtis-nflx-test/*",
"arn:aws:s3:::curtis-nflx-test"
],
"Sid": "cmccastrapel159494014dsd1shak"
},
{
"Action": [
"ec2:describevolumes",
"ec2:detachvolume",
"ec2:describelicenses",
"ec2:AssignIpv6Addresses",
"ec2:reportinstancestatus"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Sid": "cmccastrapel1594940141hlvvv"
},
{
"Action": [
"sts:AssumeRole"
],
"Effect": "Allow",
"Resource": [
"arn:aws:iam::123456789012:role/curtisTestInstanceProfile"
],
"Sid": "cmccastrapel1596483596easdits"
}
]
}
}
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "assume_role_policy",
"policy": {
"policy_document": {
"Statement": [
{
"Action": "sts:AssumeRole",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::123456789012:role/consolemeInstanceProfile"
},
"Sid": "AllowConsoleMeProdAssumeRolses"
}
],
"Version": "2012-10-17"
}
}
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "managed_policy",
"policy_name": "ApiProtect",
"action": "attach",
"arn": "arn:aws:iam::123456789012:policy/ApiProtect"
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "managed_policy",
"policy_name": "TagProtect",
"action": "detach",
"arn": "arn:aws:iam::123456789012:policy/TagProtect"
},
{
"principal_arn": "arn:aws:iam::123456789012:role/curtisTestRole1",
"change_type": "inline_policy",
"policy_name": "random_policy254",
"action": "attach",
"policy": {
"policy_document": {
"Version": "2012-10-17",
"Statement": [
{
"Action": [
"ec2:AssignIpv6Addresses"
],
"Effect": "Allow",
"Resource": [
"*"
],
"Sid": "cmccastrapel1594940141shakabcd"
}
]
}
}
}
]
},
"justification": "testing this out.",
"admin_auto_approve": false
}
Response example JSON: (Response Schema is RequestCreationResponse in models.py)
{
"errors": 1,
"request_created": true,
"request_id": "0c9fb298-c8ea-4d50-917c-3212da07b3ad",
"action_results": [
{
"status": "success",
"message": "Success description"
},
{
"status": "error",
"message": "Error description"
}
]
}
"""
if config.get("policy_editor.disallow_contractors",
True) and self.contractor:
if self.user not in config.get(
"groups.can_bypass_contractor_restrictions", []):
raise MustBeFte("Only FTEs are authorized to view this page.")
tags = {"user": self.user}
stats.count("RequestHandler.post", tags=tags)
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"user": self.user,
"message": "Create request initialization",
"user-agent": self.request.headers.get("User-Agent"),
"request_id": self.request_uuid,
"ip": self.ip,
"admin_auto_approved": False,
"probe_auto_approved": False,
}
log.debug(log_data)
try:
# Validate the model
changes = RequestCreationModel.parse_raw(self.request.body)
extended_request = await generate_request_from_change_model_array(
changes, self.user)
log_data["request"] = extended_request.dict()
log.debug(log_data)
admin_approved = False
approval_probe_approved = False
# TODO: Provide a note to the requester that admin_auto_approve will apply the requested policies only.
# It will not automatically apply generated policies. The administrative user will need to visit the policy
# Request page to do this manually.
if changes.admin_auto_approve:
# make sure user is allowed to use admin_auto_approve
can_manage_policy_request = (can_admin_policies(
self.user, self.groups), )
if can_manage_policy_request:
extended_request.request_status = RequestStatus.approved
admin_approved = True
extended_request.reviewer = self.user
self_approval_comment = CommentModel(
id=str(uuid.uuid4()),
timestamp=int(time.time()),
user_email=self.user,
user=extended_request.requester_info,
last_modified=int(time.time()),
text=f"Self-approved by admin: {self.user}",
)
extended_request.comments.append(self_approval_comment)
log_data["admin_auto_approved"] = True
log_data["request"] = extended_request.dict()
log.debug(log_data)
stats.count(
f"{log_data['function']}.post.admin_auto_approved",
tags={"user": self.user},
)
else:
# someone is trying to use admin bypass without being an admin, don't allow request to proceed
stats.count(
f"{log_data['function']}.post.unauthorized_admin_bypass",
tags={"user": self.user},
)
log_data[
"message"] = "Unauthorized user trying to use admin bypass"
log.error(log_data)
await write_json_error("Unauthorized", obj=self)
return
else:
# If admin auto approve is false, check for auto-approve probe eligibility
is_eligible_for_auto_approve_probe = (
await is_request_eligible_for_auto_approval(
extended_request, self.user))
# If we have only made requests that are eligible for auto-approval probe, check against them
if is_eligible_for_auto_approve_probe:
should_auto_approve_request = await should_auto_approve_policy_v2(
extended_request, self.user, self.groups)
if should_auto_approve_request["approved"]:
extended_request.request_status = RequestStatus.approved
approval_probe_approved = True
stats.count(
f"{log_data['function']}.probe_auto_approved",
tags={"user": self.user},
)
approving_probes = []
for approving_probe in should_auto_approve_request[
"approving_probes"]:
approving_probe_comment = CommentModel(
id=str(uuid.uuid4()),
timestamp=int(time.time()),
user_email=
f"Auto-Approve Probe: {approving_probe['name']}",
last_modified=int(time.time()),
text=
f"Policy {approving_probe['policy']} auto-approved by probe: {approving_probe['name']}",
)
extended_request.comments.append(
approving_probe_comment)
approving_probes.append(approving_probe["name"])
extended_request.reviewer = (
f"Auto-Approve Probe: {','.join(approving_probes)}"
)
log_data["probe_auto_approved"] = True
log_data["request"] = extended_request.dict()
log.debug(log_data)
dynamo = UserDynamoHandler(self.user)
request = await dynamo.write_policy_request_v2(extended_request)
log_data["message"] = "New request created in Dynamo"
log_data["request"] = extended_request.dict()
log_data["dynamo_request"] = request
log.debug(log_data)
except (InvalidRequestParameter, ValidationError) as e:
log_data["message"] = "Validation Exception"
log.error(log_data, exc_info=True)
stats.count(f"{log_data['function']}.validation_exception",
tags={"user": self.user})
self.write_error(400, message="Error validating input: " + str(e))
if config.get("development"):
raise
return
except Exception as e:
log_data[
"message"] = "Unknown Exception occurred while parsing request"
log.error(log_data, exc_info=True)
stats.count(f"{log_data['function']}.exception",
tags={"user": self.user})
sentry_sdk.capture_exception(tags={"user": self.user})
self.write_error(500, message="Error parsing request: " + str(e))
if config.get("development"):
raise
return
# If here, request has been successfully created
response = RequestCreationResponse(
errors=0,
request_created=True,
request_id=extended_request.id,
request_url=f"/policies/request/{extended_request.id}",
action_results=[],
)
# If approved is true due to an auto-approval probe or admin auto-approval, apply the non-autogenerated changes
if extended_request.request_status == RequestStatus.approved:
for change in extended_request.changes.changes:
if change.autogenerated:
continue
policy_request_modification_model = (
PolicyRequestModificationRequestModel.parse_obj({
"modification_model": {
"command": "apply_change",
"change_id": change.id,
}
}))
policy_apply_response = (
await parse_and_apply_policy_request_modification(
extended_request,
policy_request_modification_model,
self.user,
self.groups,
int(time.time()),
approval_probe_approved,
))
response.errors = policy_apply_response.errors
response.action_results = policy_apply_response.action_results
# Update in dynamo
await dynamo.write_policy_request_v2(extended_request)
account_id = await get_resource_account(extended_request.arn)
# Force a refresh of the role in Redis/DDB
arn_parsed = parse_arn(extended_request.arn)
if arn_parsed["service"] == "iam" and arn_parsed[
"resource"] == "role":
await aws.fetch_iam_role(account_id,
extended_request.arn,
force_refresh=True)
log_data["request"] = extended_request.dict()
log_data["message"] = "Applied changes based on approved request"
log_data["response"] = response.dict()
log.debug(log_data)
await aws.send_communications_new_policy_request(
extended_request, admin_approved, approval_probe_approved)
self.write(response.json())
await self.finish()
return
def make_app(jwt_validator=None):
"""make_app."""
path = pkg_resources.resource_filename("consoleme", "templates")
oss_routes = [
(r"/auth", AuthHandler),
(r"/healthcheck", HealthHandler),
(
r"/static/(.*)",
tornado.web.StaticFileHandler,
dict(path=os.path.join(path, "static")),
),
(
r"/images/(.*)",
tornado.web.StaticFileHandler,
dict(path=os.path.join(path, "images")),
),
(
r"/(favicon.ico)",
tornado.web.StaticFileHandler,
dict(path=path),
),
(r"/api/v1/get_credentials", GetCredentialsHandler),
(r"/api/v1/get_roles", GetRolesHandler),
(r"/api/v2/get_resource_url", GetResourceURLHandler),
# Used to autocomplete AWS permissions
(r"/api/v1/policyuniverse/autocomplete/?", AutocompleteHandler),
(r"/api/v2/user_profile/?", UserProfileHandler),
(r"/api/v2/self_service_config/?", SelfServiceConfigHandler),
(r"/api/v2/permission_templates/?", PermissionTemplatesHandler),
(r"/api/v1/myheaders/?", ApiHeaderHandler),
(r"/api/v1/policies/typeahead", ApiResourceTypeAheadHandler),
(r"/api/v2/policies/check", CheckPoliciesHandler),
(r"/api/v2/dynamic_config", DynamicConfigApiHandler),
(r"/api/v2/eligible_roles", EligibleRoleHandler),
(r"/api/v2/eligible_roles_page_config", EligibleRolePageConfigHandler),
(r"/api/v2/policies_page_config", PoliciesPageConfigHandler),
(r"/api/v2/requests_page_config", RequestsPageConfigHandler),
(r"/api/v2/generate_policy", GeneratePolicyHandler),
(r"/api/v2/managed_policies/(\d{12})",
ManagedPoliciesForAccountHandler),
(r"/api/v2/managed_policies/(.*)", ManagedPoliciesHandler),
(
r"/api/v2/templated_resource/([a-zA-Z0-9_-]+)/(.*)",
TemplatedResourceDetailHandler,
),
(
r"/api/v2/managed_policies_on_role/(\d{12})/(.*)",
ManagedPoliciesOnRoleHandler,
),
(r"/api/v2/login", LoginHandler),
(r"/api/v2/login_configuration", LoginConfigurationHandler),
(r"/api/v2/logout", LogOutHandler),
(
r"/api/v2/typeahead/self_service_resources",
SelfServiceStep1ResourceTypeahead,
),
(r"/api/v2/user", UserManagementHandler),
(r"/api/v2/user_registration", UserRegistrationHandler),
(r"/api/v2/policies", PoliciesHandler),
(r"/api/v2/request", RequestHandler),
(r"/api/v2/requests", RequestsHandler),
(r"/api/v2/requests/([a-zA-Z0-9_-]+)", RequestDetailHandler),
(r"/api/v2/roles/?", RolesHandler),
(r"/api/v2/roles/(\d{12})", AccountRolesHandler),
(r"/api/v2/roles/(\d{12})/(.*)", RoleDetailHandler),
(
r"/api/v2/resources/(\d{12})/(s3|sqs|sns)(?:/([a-z\-1-9]+))?/(.*)",
ResourceDetailHandler,
),
(r"/api/v2/service_control_policies/(.*)",
ServiceControlPolicyHandler),
(r"/api/v2/mtls/roles/(\d{12})/(.*)", RoleDetailAppHandler),
(r"/api/v2/clone/role", RoleCloneHandler),
(r"/api/v2/generate_changes/?", GenerateChangesHandler),
(r"/api/v2/typeahead/resources", ResourceTypeAheadHandlerV2),
(r"/api/v2/role_login/(.*)", RoleConsoleLoginHandler),
(r"/myheaders/?", HeaderHandler),
(r"/policies/typeahead/?", ResourceTypeAheadHandler),
(r"/saml/(.*)", SamlHandler),
(
r"/api/v2/challenge_validator/([a-zA-Z0-9_-]+)",
ChallengeValidatorHandler,
),
(r"/noauth/v1/challenge_generator/(.*)", ChallengeGeneratorHandler),
(r"/noauth/v1/challenge_poller/([a-zA-Z0-9_-]+)",
ChallengePollerHandler),
(r"/api/v2/.*", V2NotFoundHandler),
(
r"/(.*)",
FrontendHandler,
dict(path=path, default_filename="index.html"),
),
]
# Prioritize internal routes before OSS routes so that OSS routes can be overridden if desired.
internal_route_list = internal_routes.get_internal_routes(
make_jwt_validator, jwt_validator)
routes = internal_route_list + oss_routes
app = tornado.web.Application(
routes,
debug=config.get("tornado.debug", False),
xsrf_cookies=config.get("tornado.xsrf", True),
xsrf_cookie_kwargs=config.get("tornado.xsrf_cookie_kwargs", {}),
template_path=config.get(
"tornado.template_path",
f"{os.path.dirname(consoleme.__file__)}/templates"),
ui_modules=internal_routes.ui_modules,
)
sentry_dsn = config.get("sentry.dsn")
if sentry_dsn:
sentry_sdk.init(
dsn=sentry_dsn,
integrations=[
TornadoIntegration(),
AioHttpIntegration(),
RedisIntegration(),
],
)
return app
is_request_eligible_for_auto_approval,
parse_and_apply_policy_request_modification,
populate_cross_account_resource_policies,
populate_old_policies,
)
from consoleme.models import (
CommentModel,
DataTableResponse,
ExtendedRequestModel,
PolicyRequestModificationRequestModel,
RequestCreationModel,
RequestCreationResponse,
RequestStatus,
)
stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
log = config.get_logger()
aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
class RequestHandler(BaseAPIV2Handler):
"""Handler for /api/v2/request
Allows for creation of a request.
"""
allowed_methods = ["POST"]
def on_finish(self) -> None:
if self.request.method != "POST":
return
async def get(self, request_id):
"""
GET /api/v2/requests/{request_id}
"""
tags = {"user": self.user}
stats.count("RequestDetailHandler.get", tags=tags)
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"user": self.user,
"message": "Get request details",
"user-agent": self.request.headers.get("User-Agent"),
"request_id": self.request_uuid,
"policy_request_id": request_id,
}
log.debug(log_data)
if config.get("policy_editor.disallow_contractors",
True) and self.contractor:
if self.user not in config.get(
"groups.can_bypass_contractor_restrictions", []):
self.write_error(
403, message="Only FTEs are authorized to view this page.")
return
try:
extended_request, last_updated = await self._get_extended_request(
request_id, log_data)
except InvalidRequestParameter as e:
sentry_sdk.capture_exception(tags={"user": self.user})
self.write_error(400, message="Error validating input: " + str(e))
return
except NoMatchingRequest as e:
sentry_sdk.capture_exception(tags={"user": self.user})
self.write_error(404, message="Error getting request:" + str(e))
return
# Run these tasks concurrently.
concurrent_results = await asyncio.gather(
populate_old_policies(extended_request, self.user),
populate_cross_account_resource_policies(extended_request,
self.user),
)
extended_request = concurrent_results[0]
populate_cross_account_resource_policies_result = concurrent_results[1]
if populate_cross_account_resource_policies_result["changed"]:
extended_request = populate_cross_account_resource_policies_result[
"extended_request"]
# Update in dynamo with the latest resource policy changes
dynamo = UserDynamoHandler(self.user)
updated_request = await dynamo.write_policy_request_v2(
extended_request)
last_updated = updated_request.get("last_updated")
can_approve_reject = (can_admin_policies(self.user, self.groups), )
can_update_cancel = await can_update_cancel_requests_v2(
extended_request.requester_email, self.user, self.groups)
can_move_back_to_pending = await can_move_back_to_pending_v2(
extended_request, last_updated, self.user, self.groups)
# In the future request_specific_config will have specific approvers for specific changes based on ABAC
request_specific_config = {
"can_approve_reject": can_approve_reject,
"can_update_cancel": can_update_cancel,
"can_move_back_to_pending": can_move_back_to_pending,
}
template = None
# Force a refresh of the role in Redis/DDB
arn_parsed = parse_arn(extended_request.arn)
if arn_parsed["service"] == "iam" and arn_parsed["resource"] == "role":
iam_role = await aws.fetch_iam_role(arn_parsed["account"],
extended_request.arn)
template = iam_role.get("templated")
response = {
"request": extended_request.json(),
"last_updated": last_updated,
"request_config": request_specific_config,
"template": template,
}
self.write(response)
async def clone_iam_role(clone_model: CloneRoleRequestModel, username):
"""
Clones IAM role within same account or across account, always creating and attaching instance profile if one exists
on the source role.
;param username: username of user requesting action
;:param clone_model: CloneRoleRequestModel, which has the following attributes:
account_id: source role's account ID
role_name: source role's name
dest_account_id: destination role's account ID (may be same as account_id)
dest_role_name: destination role's name
clone_options: dict to indicate what to copy when cloning:
assume_role_policy: bool
default: False - uses default ConsoleMe AssumeRolePolicy
tags: bool
default: False - defaults to no tags
copy_description: bool
default: False - defaults to copying provided description or default description
description: string
default: "Role cloned via ConsoleMe by `username` from `arn:aws:iam::<account_id>:role/<role_name>`
if copy_description is True, then description is ignored
inline_policies: bool
default: False - defaults to no inline policies
managed_policies: bool
default: False - defaults to no managed policies
:return: results: - indicating the results of each action
"""
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Attempting to clone role",
"account_id": clone_model.account_id,
"role_name": clone_model.role_name,
"dest_account_id": clone_model.dest_account_id,
"dest_role_name": clone_model.dest_role_name,
"user": username,
}
log.info(log_data)
role = await fetch_role_details(clone_model.account_id, clone_model.role_name)
default_trust_policy = config.get("user_role_creator.default_trust_policy")
trust_policy = (
role.assume_role_policy_document
if clone_model.options.assume_role_policy
else default_trust_policy
)
if trust_policy is None:
raise MissingConfigurationValue(
"Missing Default Assume Role Policy Configuration"
)
if (
clone_model.options.copy_description
and role.description is not None
and role.description != ""
):
description = role.description
elif (
clone_model.options.description is not None
and clone_model.options.description != ""
):
description = clone_model.options.description
else:
description = f"Role cloned via ConsoleMe by {username} from {role.arn}"
tags = role.tags if clone_model.options.tags and role.tags else []
iam_client = await sync_to_async(boto3_cached_conn)(
"iam",
service_type="client",
account_number=clone_model.dest_account_id,
region=config.region,
assume_role=config.get("policies.role_name"),
session_name="clone_role_" + username,
)
results = {"errors": 0, "role_created": "false", "action_results": []}
try:
await sync_to_async(iam_client.create_role)(
RoleName=clone_model.dest_role_name,
AssumeRolePolicyDocument=json.dumps(trust_policy),
Description=description,
Tags=tags,
)
results["action_results"].append(
{
"status": "success",
"message": f"Role arn:aws:iam::{clone_model.dest_account_id}:role/{clone_model.dest_role_name} "
f"successfully created",
}
)
results["role_created"] = "true"
except Exception as e:
log_data["message"] = "Exception occurred creating cloned role"
log_data["error"] = str(e)
log.error(log_data, exc_info=True)
results["action_results"].append(
{
"status": "error",
"message": f"Error creating role {clone_model.dest_role_name} in account {clone_model.dest_account_id}:"
+ str(e),
}
)
results["errors"] += 1
sentry_sdk.capture_exception()
# Since we were unable to create the role, no point continuing, just return
return results
if clone_model.options.tags:
results["action_results"].append(
{"status": "success", "message": "Successfully copied tags"}
)
if clone_model.options.assume_role_policy:
results["action_results"].append(
{
"status": "success",
"message": "Successfully copied Assume Role Policy Document",
}
)
else:
results["action_results"].append(
{
"status": "success",
"message": "Successfully added default Assume Role Policy Document",
}
)
if (
clone_model.options.copy_description
and role.description is not None
and role.description != ""
):
results["action_results"].append(
{"status": "success", "message": "Successfully copied description"}
)
elif clone_model.options.copy_description:
results["action_results"].append(
{
"status": "error",
"message": "Failed to copy description, so added default description: "
+ description,
}
)
else:
results["action_results"].append(
{
"status": "success",
"message": "Successfully added description: " + description,
}
)
# Create instance profile and attach if it exists in source role
if len(list(await sync_to_async(role.instance_profiles.all)())) > 0:
try:
await sync_to_async(iam_client.create_instance_profile)(
InstanceProfileName=clone_model.dest_role_name
)
await sync_to_async(iam_client.add_role_to_instance_profile)(
InstanceProfileName=clone_model.dest_role_name,
RoleName=clone_model.dest_role_name,
)
results["action_results"].append(
{
"status": "success",
"message": f"Successfully added instance profile {clone_model.dest_role_name} to role "
f"{clone_model.dest_role_name}",
}
)
except Exception as e:
log_data[
"message"
] = "Exception occurred creating/attaching instance profile"
log_data["error"] = str(e)
log.error(log_data, exc_info=True)
sentry_sdk.capture_exception()
results["action_results"].append(
{
"status": "error",
"message": f"Error creating/attaching instance profile {clone_model.dest_role_name} to role: "
+ str(e),
}
)
results["errors"] += 1
# other optional attributes to copy over after role has been successfully created
cloned_role = await fetch_role_details(
clone_model.dest_account_id, clone_model.dest_role_name
)
# Copy inline policies
if clone_model.options.inline_policies:
for src_policy in await sync_to_async(role.policies.all)():
await sync_to_async(src_policy.load)()
try:
dest_policy = await sync_to_async(cloned_role.Policy)(src_policy.name)
await sync_to_async(dest_policy.put)(
PolicyDocument=json.dumps(src_policy.policy_document)
)
results["action_results"].append(
{
"status": "success",
"message": f"Successfully copied inline policy {src_policy.name}",
}
)
except Exception as e:
log_data["message"] = "Exception occurred copying inline policy"
log_data["error"] = str(e)
log.error(log_data, exc_info=True)
sentry_sdk.capture_exception()
results["action_results"].append(
{
"status": "error",
"message": f"Error copying inline policy {src_policy.name}: "
+ str(e),
}
)
results["errors"] += 1
# Copy managed policies
if clone_model.options.managed_policies:
for src_policy in await sync_to_async(role.attached_policies.all)():
await sync_to_async(src_policy.load)()
dest_policy_arn = src_policy.arn.replace(
clone_model.account_id, clone_model.dest_account_id
)
try:
await sync_to_async(cloned_role.attach_policy)(
PolicyArn=dest_policy_arn
)
results["action_results"].append(
{
"status": "success",
"message": f"Successfully attached managed policy {src_policy.arn} as {dest_policy_arn}",
}
)
except Exception as e:
log_data["message"] = "Exception occurred copying managed policy"
log_data["error"] = str(e)
log.error(log_data, exc_info=True)
sentry_sdk.capture_exception()
results["action_results"].append(
{
"status": "error",
"message": f"Error attaching managed policy {dest_policy_arn}: "
+ str(e),
}
)
results["errors"] += 1
stats.count(
f"{log_data['function']}.success", tags={"role_name": clone_model.role_name}
)
log_data["message"] = "Successfully cloned role"
log.info(log_data)
return results
async def fetch_iam_role(self,
account_id: str,
role_arn: str,
force_refresh: bool = False) -> dict:
"""Fetch the IAM Role template from Redis and/or Dynamo.
:param account_id:
:param role_arn:
:return:
"""
log_data: dict = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"role_arn": role_arn,
"account_id": account_id,
"force_refresh": force_refresh,
}
result: dict = {}
if not force_refresh:
# First check redis:
result: str = await sync_to_async(self._fetch_role_from_redis
)(role_arn)
if result:
result: dict = json.loads(result)
# If this item is less than an hour old, then return it from Redis.
if result["ttl"] > int(
(datetime.utcnow() - timedelta(hours=1)).timestamp()):
log_data[
"message"] = "Role not in Redis -- fetching from DDB."
log.debug(log_data)
stats.count(
"aws.fetch_iam_role.in_redis",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
result["policy"] = json.loads(result["policy"])
return result
# If not in Redis or it's older than an hour, proceed to DynamoDB:
result = await sync_to_async(self.dynamo.fetch_iam_role
)(role_arn, account_id)
# If it's NOT in dynamo, or if we're forcing a refresh, we need to reach out to AWS and fetch:
if force_refresh or not result.get("Item"):
if force_refresh:
log_data[
"message"] = "Force refresh is enabled. Going out to AWS."
stats.count(
"aws.fetch_iam_role.force_refresh",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
else:
log_data[
"message"] = "Role is missing in DDB. Going out to AWS."
stats.count(
"aws.fetch_iam_role.missing_dynamo",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
log.debug(log_data)
try:
tasks = []
role_name = role_arn.split("/")[-1]
# Instantiate a cached CloudAux client
client = await sync_to_async(boto3_cached_conn)(
"iam",
account_number=account_id,
assume_role=config.get("policies.role_name"),
)
conn = {
"account_number": account_id,
"assume_role": config.get("policies.role_name"),
"region": config.region,
}
role_details = asyncio.ensure_future(
sync_to_async(client.get_role)(RoleName=role_name))
tasks.append(role_details)
all_tasks = [
get_role_managed_policies,
get_role_inline_policies,
list_role_tags,
]
for t in all_tasks:
tasks.append(
asyncio.ensure_future(
sync_to_async(t)({
"RoleName": role_name
}, **conn)))
responses = asyncio.gather(*tasks)
result = await responses
role = result[0]["Role"]
role["ManagedPolicies"] = result[1]
role["InlinePolicies"] = result[2]
role["Tags"] = result[3]
except ClientError as ce:
if ce.response["Error"]["Code"] == "NoSuchEntity":
# The role does not exist:
log_data["message"] = "Role does not exist in AWS."
log.error(log_data)
stats.count(
"aws.fetch_iam_role.missing_in_aws",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
return None
else:
log_data["message"] = f"Some other error: {ce.response}"
log.error(log_data)
stats.count(
"aws.fetch_iam_role.aws_connection_problem",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
raise
# Format the role for DynamoDB and Redis:
await self._cloudaux_to_aws(role)
result = {
"arn":
role.get("Arn"),
"name":
role.pop("RoleName"),
"resourceId":
role.pop("RoleId"),
"accountId":
account_id,
"ttl":
int((datetime.utcnow() + timedelta(hours=36)).timestamp()),
"policy":
self.dynamo.convert_role_to_json(role),
"templated":
self.red.hget(
config.get("templated_roles.redis_key",
"TEMPLATED_ROLES_v2"),
role.get("Arn").lower(),
),
}
# Sync with DDB:
await sync_to_async(self.dynamo.sync_iam_role_for_account)(result)
log_data["message"] = "Role fetched from AWS, and synced with DDB."
stats.count(
"aws.fetch_iam_role.fetched_from_aws",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
else:
log_data["message"] = "Role fetched from DDB."
stats.count(
"aws.fetch_iam_role.in_dynamo",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
# Fix the TTL:
result["Item"]["ttl"] = int(result["Item"]["ttl"])
result = result["Item"]
# Update the redis cache:
stats.count(
"aws.fetch_iam_role.in_dynamo",
tags={
"account_id": account_id,
"role_arn": role_arn
},
)
await sync_to_async(self._add_role_to_redis)(result)
log_data["message"] += " Updated Redis."
log.debug(log_data)
result["policy"] = json.loads(result["policy"])
return result
def main():
if config.get("sso.create_mock_jwk"):
app = make_app(jwt_validator=lambda x: {})
else:
app = make_app()
return app
async def fetch_s3_bucket(account_id: str, bucket_name: str) -> dict:
"""Fetch S3 Bucket and applicable policies
:param account_id:
:param bucket_name:
:return:
"""
log_data: Dict = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"bucket_name": bucket_name,
"account_id": account_id,
}
log.debug(log_data)
created_time = None
try:
bucket_resource = await sync_to_async(get_bucket_resource)(
bucket_name,
account_number=account_id,
assume_role=config.get("policies.role_name"),
region=config.region,
sts_client_kwargs=dict(
region_name=config.region,
endpoint_url=f"https://sts.{config.region}.amazonaws.com",
),
)
created_time_stamp = bucket_resource.creation_date
if created_time_stamp:
created_time = created_time_stamp.isoformat()
except ClientError:
sentry_sdk.capture_exception()
try:
bucket_location = await get_bucket_location_with_fallback(
bucket_name, account_id
)
policy: Dict = await sync_to_async(get_bucket_policy)(
account_number=account_id,
assume_role=config.get("policies.role_name"),
region=bucket_location,
Bucket=bucket_name,
sts_client_kwargs=dict(
region_name=config.region,
endpoint_url=f"https://sts.{config.region}.amazonaws.com",
),
)
except ClientError as e:
if "NoSuchBucketPolicy" in str(e):
policy = {"Policy": "{}"}
else:
raise
try:
tags: Dict = await sync_to_async(get_bucket_tagging)(
account_number=account_id,
assume_role=config.get("policies.role_name"),
region=bucket_location,
Bucket=bucket_name,
sts_client_kwargs=dict(
region_name=config.region,
endpoint_url=f"https://sts.{config.region}.amazonaws.com",
),
)
except ClientError as e:
if "NoSuchTagSet" in str(e):
tags = {"TagSet": []}
else:
raise
result: Dict = {**policy, **tags, "created_time": created_time}
result["Policy"] = json.loads(result["Policy"])
return result
import asyncio
import logging
import os
import tornado.autoreload
import tornado.httpserver
import tornado.ioloop
import uvloop
from tornado.platform.asyncio import AsyncIOMainLoop
from consoleme.config import config
from consoleme.lib.plugins import get_plugin_by_name
from consoleme.routes import make_app
logging.basicConfig(level=logging.DEBUG, format=config.get("logging.format"))
logging.getLogger("urllib3.connectionpool").setLevel(logging.CRITICAL)
stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
log = config.get_logger()
def main():
if config.get("sso.create_mock_jwk"):
app = make_app(jwt_validator=lambda x: {})
else:
app = make_app()
return app
if config.get("tornado.uvloop", True):
asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
async def get_role_template(arn: str):
return await sync_to_async(red.hget)(
config.get("templated_roles.redis_key", "TEMPLATED_ROLES_v2"), arn.lower()
)
from consoleme.lib.crypto import Crypto
from consoleme.lib.plugins import get_plugin_by_name
from consoleme.lib.policies import get_aws_config_history_url_for_resource
from consoleme.lib.redis import RedisHandler, redis_get
from consoleme.models import (
CloudTrailDetailsModel,
CloudTrailError,
CloudTrailErrorArray,
ExtendedRoleModel,
RoleModel,
S3DetailsModel,
S3Error,
S3ErrorArray,
)
stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
log = config.get_logger()
crypto = Crypto()
auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
aws = get_plugin_by_name(config.get("plugins.aws", "default_aws"))()
internal_policies = get_plugin_by_name(
config.get("plugins.internal_policies", "default_policies")
)()
red = RedisHandler().redis_sync()
async def get_config_timeline_url_for_role(role, account_id):
resource_id = role.get("resourceId")
if resource_id:
config_history_url = await get_aws_config_history_url_for_resource(
account_id, resource_id, role["arn"], "AWS::IAM::Role"
def test_clone_authorized_user(self, mock_can_create_roles):
import boto3
from consoleme.config import config
mock_can_create_roles.return_value = True
input_body = {
"dest_account_id": "012345678901",
"dest_role_name": "testing_dest_role",
"account_id": "012345678901",
"options": {
"tags": "False",
"inline_policies": "True",
"assume_role_policy": "True",
"copy_description": "False",
"description": "Testing this should appear",
},
}
expected = {
"status":
400,
"title":
"Bad Request",
"message":
"Error validating input: 1 validation error for CloneRoleRequestModel\nRoleName\n "
"field required (type=value_error.missing)",
}
response = self.fetch("/api/v2/clone/role",
method="POST",
body=json.dumps(input_body))
self.assertEqual(response.code, 400)
self.assertDictEqual(json.loads(response.body), expected)
client = boto3.client("iam",
region_name="us-east-1",
**config.get("boto3.client_kwargs", {}))
role_name = "fake_account_admin"
client.create_role(
RoleName=role_name,
AssumeRolePolicyDocument="{}",
Description="Should not appear",
)
client.create_instance_profile(
InstanceProfileName="testinstanceprofile")
client.add_role_to_instance_profile(
InstanceProfileName="testinstanceprofile", RoleName=role_name)
input_body["role_name"] = role_name
expected = {
"errors":
0,
"role_created":
"true",
"action_results": [
{
"status":
"success",
"message":
"Role arn:aws:iam::012345678901:role/testing_dest_role successfully created",
},
{
"status": "success",
"message":
"Successfully copied Assume Role Policy Document",
},
{
"status":
"success",
"message":
"Successfully added description: Testing this should appear",
},
{
"status":
"success",
"message":
"Successfully added instance profile testing_dest_role to role testing_dest_role",
},
],
}
response = self.fetch("/api/v2/clone/role",
method="POST",
body=json.dumps(input_body))
self.assertEqual(response.code, 200)
self.assertDictEqual(json.loads(response.body), expected)
async def get_credentials(
self,
user: str,
role: str,
enforce_ip_restrictions: bool = True,
user_role: bool = False,
account_id: str = None,
custom_ip_restrictions: list = None,
) -> dict:
"""Get Credentials will return the list of temporary credentials from AWS."""
log_data = {
"function":
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}",
"user": user,
"role": role,
"enforce_ip_restrictions": enforce_ip_restrictions,
"custom_ip_restrictions": custom_ip_restrictions,
"message": "Generating credentials",
}
session = boto3.Session()
client = session.client("sts", region_name=config.region)
ip_restrictions = config.get("aws.ip_restrictions")
stats.count("aws.get_credentials", tags={"role": role, "user": user})
# If this is a dynamic request, then we need to fetch the role details, call out to the lambda
# wait for it to complete, assume the role, and then return the assumed credentials back.
if user_role:
stats.count("aws.call_user_lambda",
tags={
"role": role,
"user": user
})
try:
role = await self.call_user_lambda(role, user, account_id)
except Exception as e:
raise e
try:
if enforce_ip_restrictions and ip_restrictions:
policy = json.dumps(
dict(
Version="2012-10-17",
Statement=[
dict(
Effect="Deny",
Action="*",
Resource="*",
Condition=dict(NotIpAddress={
"aws:SourceIP": ip_restrictions
}),
),
dict(Effect="Allow", Action="*", Resource="*"),
],
))
credentials = await sync_to_async(client.assume_role)(
RoleArn=role,
RoleSessionName=user.lower(),
Policy=policy,
DurationSeconds=config.get("aws.session_duration", 3600),
)
credentials["Credentials"]["Expiration"] = int(
credentials["Credentials"]["Expiration"].timestamp())
return credentials
if custom_ip_restrictions:
policy = json.dumps(
dict(
Version="2012-10-17",
Statement=[
dict(
Effect="Deny",
Action="*",
Resource="*",
Condition=dict(
NotIpAddress={
"aws:SourceIP": custom_ip_restrictions
}),
),
dict(Effect="Allow", Action="*", Resource="*"),
],
))
credentials = await sync_to_async(client.assume_role)(
RoleArn=role,
RoleSessionName=user.lower(),
Policy=policy,
DurationSeconds=config.get("aws.session_duration", 3600),
)
credentials["Credentials"]["Expiration"] = int(
credentials["Credentials"]["Expiration"].timestamp())
return credentials
credentials = await sync_to_async(client.assume_role)(
RoleArn=role,
RoleSessionName=user.lower(),
DurationSeconds=config.get("aws.session_duration", 3600),
)
credentials["Credentials"]["Expiration"] = int(
credentials["Credentials"]["Expiration"].timestamp())
log.debug(log_data)
return credentials
except ClientError as e:
# TODO(ccastrapel): Determine if user role was really just created, or if this is an older role.
if user_role:
raise UserRoleNotAssumableYet(e.response["Error"])
raise
NoMatchingRequest,
PendingRequestAlreadyExists,
)
from consoleme.lib.crypto import Crypto
from consoleme.lib.plugins import get_plugin_by_name
from consoleme.lib.redis import RedisHandler
from consoleme.models import ExtendedRequestModel
DYNAMO_EMPTY_STRING = "---DYNAMO-EMPTY-STRING---"
# We need to import Decimal to eval the request. This Decimal usage is to prevent lint errors on importing the unused
# Decimal module.
DYNAMODB_EMPTY_DECIMAL = Decimal(0)
POSSIBLE_STATUSES = config.get(
"possible_statuses",
["pending", "approved", "rejected", "cancelled", "expired", "removed"],
)
stats = get_plugin_by_name(config.get("plugins.metrics", "default_metrics"))()
log = config.get_logger("consoleme")
crypto = Crypto()
red = RedisHandler().redis_sync()
class BaseDynamoHandler:
"""Base class for interacting with DynamoDB."""
def _get_dynamo_table(self, table_name):
function: str = (
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}"
)
try:
def __init__(self):
self.red = RedisHandler().redis_sync()
self.redis_key = config.get("aws.iamroles_redis_key", "IAM_ROLE_CACHE")
self.dynamo = IAMRoleDynamoHandler()
async def post(self):
"""
POST /api/v2/requests
"""
arguments = {k: self.get_argument(k) for k in self.request.arguments}
markdown = arguments.get("markdown")
cache_key = config.get("cache_all_policy_requests.redis_key",
"ALL_POLICY_REQUESTS")
s3_bucket = config.get("cache_policy_requests.s3.bucket")
s3_key = config.get(
"cache_policy_requests.s3.file",
"policy_requests/all_policy_requests_v1.json.gz",
)
arguments = json.loads(self.request.body)
filters = arguments.get("filters")
# TODO: Add server-side sorting
# sort = arguments.get("sort")
limit = arguments.get("limit", 1000)
tags = {"user": self.user}
stats.count("RequestsHandler.post", tags=tags)
log_data = {
"function": "RequestsHandler.post",
"user": self.user,
"message": "Writing requests",
"limit": limit,
"filters": filters,
"user-agent": self.request.headers.get("User-Agent"),
"request_id": self.request_uuid,
}
log.debug(log_data)
requests = await retrieve_json_data_from_redis_or_s3(
cache_key, s3_bucket=s3_bucket, s3_key=s3_key)
total_count = len(requests)
if filters:
try:
with Timeout(seconds=5):
for filter_key, filter_value in filters.items():
requests = await filter_table(filter_key, filter_value,
requests)
except TimeoutError:
self.write("Query took too long to run. Check your filter.")
await self.finish()
raise
if markdown:
requests_to_write = []
for request in requests[0:limit]:
resource_name = request["arn"].split(":")[5]
if "/" in resource_name:
resource_name = resource_name.split("/")[-1]
region = request["arn"].split(":")[3]
service_type = request["arn"].split(":")[2]
account_id = request["arn"].split(":")[4]
try:
url = await get_url_for_resource(
request["arn"],
service_type,
account_id,
region,
resource_name,
)
except ResourceNotFound:
url = None
# Convert request_id and role ARN to link
if request.get("version") == "2":
request[
"request_id"] = f"[{request['request_id']}](/policies/request/{request['request_id']})"
# Legacy support for V1 requests. Pending removal.
else:
request[
"request_id"] = f"[{request['request_id']}](/policies/request_v1/{request['request_id']})"
if url:
request["arn"] = f"[{request['arn']}]({url})"
requests_to_write.append(request)
else:
requests_to_write = requests[0:limit]
filtered_count = len(requests_to_write)
res = DataTableResponse(totalCount=total_count,
filteredCount=filtered_count,
data=requests_to_write)
self.write(res.json())
return
async def generate_url(
self,
user: str,
role: str,
region: str = "us-east-1",
user_role: bool = False,
account_id: str = None,
) -> str:
"""Generate URL will get temporary credentials and craft a URL with those credentials."""
function = (
f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}"
)
log_data = {
"function": function,
"user": user,
"role": role,
"message": "Generating authenticated AWS console URL",
}
log.debug(log_data)
credentials = await self.get_credentials(
user,
role,
user_role=user_role,
account_id=account_id,
enforce_ip_restrictions=False,
)
credentials_d = {
"sessionId": credentials.get("Credentials", {}).get("AccessKeyId"),
"sessionKey": credentials.get("Credentials",
{}).get("SecretAccessKey"),
"sessionToken": credentials.get("Credentials",
{}).get("SessionToken"),
}
req_params = {
"Action": "getSigninToken",
"Session": bleach.clean(json.dumps(credentials_d)),
"DurationSeconds": config.get("aws.session_duration", 3600),
}
http_client = AsyncHTTPClient(force_instance=True)
url_with_params: str = url_concat(
config.get("aws.federation_url",
"https://signin.aws.amazon.com/federation"),
req_params,
)
r = await http_client.fetch(url_with_params,
ssl_options=ssl.SSLContext())
token = json.loads(r.body)
login_req_params = {
"Action":
"login",
"Issuer":
config.get("aws.issuer"),
"Destination": ("{}".format(
config.get(
"aws.console_url",
"https://{}.console.aws.amazon.com").format(region))),
"SigninToken":
bleach.clean(token.get("SigninToken")),
"SessionDuration":
config.get("aws.session_duration", 3600),
}
r2 = requests_sync.Request(
"GET",
config.get("aws.federation_url",
"https://signin.aws.amazon.com/federation"),
params=login_req_params,
)
url = r2.prepare().url
return url
async def get(self):
"""
/requests_page_config
---
get:
description: Retrieve Requests Page Configuration
responses:
200:
description: Returns Requests Page Configuration
"""
default_configuration = {
"pageName": "Requests",
"pageDescription":
"View all IAM policy requests created through ConsoleMe",
"tableConfig": {
"expandableRows":
True,
"dataEndpoint":
"/api/v2/requests?markdown=true",
"sortable":
False,
"totalRows":
200,
"rowsPerPage":
50,
"serverSideFiltering":
True,
"allowCsvExport":
True,
"allowJsonExport":
True,
"columns": [
{
"placeholder": "Username",
"key": "username",
"type": "input",
"style": {
"width": "100px"
},
},
{
"placeholder": "Arn",
"key": "arn",
"type": "link",
"style": {
"whiteSpace": "normal",
"wordBreak": "break-all"
},
"width": 3,
},
{
"placeholder": "Request Time",
"key": "request_time",
"type": "daterange",
},
{
"placeholder": "Status",
"key": "status",
"type": "dropdown",
"style": {
"width": "90px"
},
},
{
"placeholder": "Request ID",
"key": "request_id",
"type": "link",
"style": {
"whiteSpace": "normal",
"wordBreak": "break-all"
},
"width": 2,
},
],
},
}
table_configuration = config.get(
"RequestsTableConfigHandler.configuration", default_configuration)
self.write(table_configuration)
from consoleme.lib.defaults import SELF_SERVICE_IAM_DEFAULTS
from consoleme.lib.generic import generate_random_string, iterate_and_format_dict
from consoleme.lib.plugins import get_plugin_by_name
from consoleme.models import (
ChangeGeneratorModel,
ChangeGeneratorModelArray,
ChangeModelArray,
CrudChangeGeneratorModel,
InlinePolicyChangeModel,
PolicyModel,
ResourceModel,
Status,
)
group_mapping = get_plugin_by_name(
config.get("plugins.group_mapping", "default_group_mapping"))()
ALL_ACCOUNTS = None
self_service_iam_config: dict = config.get("self_service_iam",
SELF_SERVICE_IAM_DEFAULTS)
async def _generate_policy_statement(actions: List, resources: List,
effect: str, condition: Dict) -> Dict:
"""
Generates an IAM policy resource given actions, effects, resources, and conditions
:param actions: a List of actions
:param resources: a List of AWS resource ARNs or wildcards
:param effect: an Effect (Allow|Deny)
:return:
"""
from consoleme.handlers.v2.typeahead import (
ResourceTypeAheadHandlerV2,
SelfServiceStep1ResourceTypeahead,
)
from consoleme.handlers.v2.user import (
LoginConfigurationHandler,
LoginHandler,
UserManagementHandler,
UserRegistrationHandler,
)
from consoleme.handlers.v2.user_profile import UserProfileHandler
from consoleme.lib.auth import mk_jwks_validator
from consoleme.lib.plugins import get_plugin_by_name
internal_routes = get_plugin_by_name(
config.get("plugins.internal_routes", "default_internal_routes"))()
log = config.get_logger()
def make_jwt_validator():
jwk_url = config.get("sso.jwk_url")
if not jwk_url:
raise Exception("Config 'sso.jwk_url' is not defined")
jwk_set = requests.get(jwk_url).json()
keys = [k for k in jwk_set["keys"] if k["kty"] == "RSA"]
jwk_schema = config.get("sso.jwk_schema")
if not jwk_schema:
raise Exception("Config 'sso.jwk_schema' is not defined")
return mk_jwks_validator(keys, jwk_schema["header"], jwk_schema["payload"])
async def store_json_results_in_redis_and_s3(
data: Union[Dict[str, set], Dict[str, str],
List[Union[Dict[str, Union[Union[str, int], Any]],
Dict[str, Union[Union[str, None, int],
Any]], ]], str, Dict[str, list], ],
redis_key: str = None,
redis_data_type: str = "str",
s3_bucket: str = None,
s3_key: str = None,
json_encoder=None,
):
"""
Stores data in Redis and S3, depending on configuration
:param redis_data_type: "str" or "hash", depending on how we're storing data in Redis
:param data: Python dictionary or list that will be encoded in JSON for storage
:param redis_key: Redis Key to store data to
:param s3_bucket: S3 bucket to store data
:param s3_key: S3 key to store data
:return:
"""
last_updated_redis_key = config.get(
"store_json_results_in_redis_and_s3.last_updated_redis_key",
"STORE_JSON_RESULTS_IN_REDIS_AND_S3_LAST_UPDATED",
)
function = f"{__name__}.{sys._getframe().f_code.co_name}"
last_updated = int(time.time())
stats.count(
f"{function}.called",
tags={
"redis_key": redis_key,
"s3_bucket": s3_bucket,
"s3_key": s3_key
},
)
if redis_key:
if redis_data_type == "str":
if isinstance(data, str):
red.set(redis_key, data)
else:
red.set(redis_key,
json.dumps(data, cls=SetEncoder, default=json_encoder))
elif redis_data_type == "hash":
red.hmset(redis_key, data)
else:
raise UnsupportedRedisDataType(
"Unsupported redis_data_type passed")
red.hset(last_updated_redis_key, redis_key, last_updated)
if s3_bucket and s3_key:
data_for_s3 = {"last_updated": last_updated, "data": data}
put_object(
Bucket=s3_bucket,
Key=s3_key,
Body=json.dumps(data_for_s3,
cls=SetEncoder,
default=json_encoder,
indent=2).encode(),
)
import asyncio
from mock import patch
from tornado.concurrent import Future
from tornado.testing import AsyncTestCase
from consoleme.config import config
from consoleme.exceptions.exceptions import NoMatchingRequest
from consoleme.lib.plugins import get_plugin_by_name
from tests.conftest import create_future
auth = get_plugin_by_name(config.get("plugins.auth", "default_auth"))()
Group = auth.Group
class TestRequestsLibrary(AsyncTestCase):
@patch("consoleme.lib.requests.UserDynamoHandler")
@patch("consoleme.lib.requests.auth")
def test_get_user_requests(self, mock_auth, mock_user_dynamo_handler):
from consoleme.lib.requests import get_user_requests
"""Chuck Norris has a request and is an secondary approver for group1"""
mock_user = "******"
mock_requests = [
{
"username": mock_user
},
{
"username": "******",
"group": "group1"
},
async def retrieve_json_data_from_redis_or_s3(
redis_key: str = None,
redis_data_type: str = "str",
s3_bucket: str = None,
s3_key: str = None,
cache_to_redis_if_data_in_s3: bool = True,
max_age: Optional[int] = None,
default: Optional = None,
json_object_hook: Optional = None,
json_encoder: Optional = None,
):
"""
Retrieve data from Redis as a priority. If data is unavailable in Redis, fall back to S3 and attempt to store
data in Redis for quicker retrieval later
:param redis_data_type: "str" or "hash", depending on how the data is stored in Redis
:param redis_key: Redis Key to retrieve data from
:param s3_bucket: S3 bucket to retrieve data from
:param s3_key: S3 key to retrieve data from
:param cache_to_redis_if_data_in_s3: Cache the data in Redis if the data is in S3 but not Redis
:return:
"""
function = f"{__name__}.{sys._getframe().f_code.co_name}"
last_updated_redis_key = config.get(
"store_json_results_in_redis_and_s3.last_updated_redis_key",
"STORE_JSON_RESULTS_IN_REDIS_AND_S3_LAST_UPDATED",
)
stats.count(
f"{function}.called",
tags={
"redis_key": redis_key,
"s3_bucket": s3_bucket,
"s3_key": s3_key
},
)
data = None
if redis_key:
if redis_data_type == "str":
data_s = red.get(redis_key)
if data_s:
data = json.loads(data_s, object_hook=json_object_hook)
elif redis_data_type == "hash":
data = red.hgetall(redis_key)
else:
raise UnsupportedRedisDataType(
"Unsupported redis_data_type passed")
if data and max_age:
current_time = int(time.time())
last_updated = int(red.hget(last_updated_redis_key, redis_key))
if current_time - last_updated > max_age:
raise ExpiredData(
f"Data in Redis is older than {max_age} seconds.")
# Fall back to S3 if there's no data
if not data and s3_bucket and s3_key:
s3_object = get_object(Bucket=s3_bucket, Key=s3_key)
s3_object_content = s3_object["Body"].read()
data_object = json.loads(s3_object_content,
object_hook=json_object_hook)
data = data_object["data"]
if data and max_age:
current_time = int(time.time())
last_updated = data_object["last_updated"]
if current_time - last_updated > max_age:
raise ExpiredData(
f"Data in S3 is older than {max_age} seconds.")
if redis_key and cache_to_redis_if_data_in_s3:
await store_json_results_in_redis_and_s3(
data,
redis_key=redis_key,
redis_data_type=redis_data_type,
json_encoder=json_encoder,
)
if data is not None:
return data
if default is not None:
return default
raise DataNotRetrievable("Unable to retrieve expected data.")
async def get(self):
try:
type_ahead: Optional[str] = (self.request.arguments.get(
"typeahead")[0].decode("utf-8").lower())
except TypeError:
type_ahead = None
try:
account_id: Optional[str] = self.request.arguments.get(
"account_id")[0].decode("utf-8")
except TypeError:
account_id = None
try:
resource_type: Optional[str] = self.request.arguments.get(
"resource_type")[0].decode("utf-8")
except TypeError:
resource_type = None
try:
region: Optional[str] = self.request.arguments.get(
"region")[0].decode("utf-8")
except TypeError:
region = None
try:
limit: int = self.request.arguments.get("limit")[0].decode("utf-8")
if limit:
limit = int(limit)
except TypeError:
limit = 20
try:
ui_formatted: Optional[bool] = (self.request.arguments.get(
"ui_formatted")[0].decode("utf-8").lower())
except TypeError:
ui_formatted = False
resource_redis_cache_key = config.get("aws_config_cache.redis_key",
"AWSCONFIG_RESOURCE_CACHE")
all_resource_arns = await sync_to_async(red.hkeys
)(resource_redis_cache_key)
# Fall back to DynamoDB or S3?
if not all_resource_arns:
s3_bucket = config.get("aws_config_cache_combined.s3.bucket")
s3_key = config.get(
"aws_config_cache_combined.s3.file",
"aws_config_cache_combined/aws_config_resource_cache_combined_v1.json.gz",
)
try:
all_resources = await retrieve_json_data_from_redis_or_s3(
s3_bucket=s3_bucket, s3_key=s3_key)
all_resource_arns = all_resources.keys()
await sync_to_async(red.hmset)(resource_redis_cache_key,
all_resources)
except DataNotRetrievable:
sentry_sdk.capture_exception()
all_resource_arns = []
matching = set()
for arn in all_resource_arns:
if len(matching) >= limit:
break
# ARN format: 'arn:aws:sqs:us-east-1:123456789012:resource_name'
if resource_type and resource_type != arn.split(":")[2]:
continue
if region and region != arn.split(":")[3]:
continue
if account_id and account_id != arn.split(":")[4]:
continue
if type_ahead and type_ahead in arn.lower():
matching.add(arn)
elif not type_ahead:
# Oh, you want all the things do you?
matching.add(arn)
arn_array = ArnArray.parse_obj((list(matching)))
if ui_formatted:
self.write(
json.dumps([{
"title": arn
} for arn in arn_array.__root__]))
else:
self.write(arn_array.json())
def query(query: str,
use_aggregator: bool = True,
account_id: Optional[str] = None) -> List:
resources = []
if use_aggregator:
config_client = boto3.client("config",
region_name=config.region,
**config.get("boto3.client_kwargs", {}))
configuration_aggregator_name: str = config.get(
"aws_config.configuration_aggregator.name").format(
region=config.region)
if not configuration_aggregator_name:
raise MissingConfigurationValue(
"Invalid configuration for aws_config")
response = config_client.select_aggregate_resource_config(
Expression=query,
ConfigurationAggregatorName=configuration_aggregator_name,
Limit=100,
)
for r in response.get("Results", []):
resources.append(json.loads(r))
while response.get("NextToken"):
response = config_client.select_aggregate_resource_config(
Expression=query,
ConfigurationAggregatorName=configuration_aggregator_name,
Limit=100,
NextToken=response["NextToken"],
)
for r in response.get("Results", []):
resources.append(json.loads(r))
return resources
else: # Don't use Config aggregator and instead query all the regions on an account
session = boto3.Session()
available_regions = session.get_available_regions("config")
excluded_regions = config.get(
"api_protect.exclude_regions",
[
"af-south-1", "ap-east-1", "ap-northeast-3", "eu-south-1",
"me-south-1"
],
)
regions = [x for x in available_regions if x not in excluded_regions]
for region in regions:
config_client = boto3_cached_conn(
"config",
account_number=account_id,
assume_role=config.get("policies.role_name"),
region=region,
sts_client_kwargs=dict(
region_name=config.region,
endpoint_url=f"https://sts.{config.region}.amazonaws.com",
),
client_kwargs=config.get("boto3.client_kwargs", {}),
)
try:
response = config_client.select_resource_config(
Expression=query, Limit=100)
for r in response.get("Results", []):
resources.append(json.loads(r))
# Query Config for a specific account in all regions we care about
while response.get("NextToken"):
response = config_client.select_resource_config(
Expression=query,
Limit=100,
NextToken=response["NextToken"])
for r in response.get("Results", []):
resources.append(json.loads(r))
except ClientError as e:
log.error(
{
"function":
f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Failed to query AWS Config",
"query": query,
"use_aggregator": use_aggregator,
"account_id": account_id,
"region": region,
"error": str(e),
},
exc_info=True,
)
sentry_sdk.capture_exception()
return resources
async def create_iam_role(create_model: RoleCreationRequestModel, username):
"""
Creates IAM role.
:param create_model: RoleCreationRequestModel, which has the following attributes:
account_id: destination account's ID
role_name: destination role name
description: optional string - description of the role
default: Role created by {username} through ConsoleMe
instance_profile: optional boolean - whether to create an instance profile and attach it to the role or not
default: True
:param username: username of user requesting action
:return: results: - indicating the results of each action
"""
log_data = {
"function": f"{__name__}.{sys._getframe().f_code.co_name}",
"message": "Attempting to create role",
"account_id": create_model.account_id,
"role_name": create_model.role_name,
"user": username,
}
log.info(log_data)
default_trust_policy = config.get("user_role_creator.default_trust_policy")
if default_trust_policy is None:
raise MissingConfigurationValue(
"Missing Default Assume Role Policy Configuration"
)
if create_model.description:
description = create_model.description
else:
description = f"Role created by {username} through ConsoleMe"
iam_client = await sync_to_async(boto3_cached_conn)(
"iam",
service_type="client",
account_number=create_model.account_id,
region=config.region,
assume_role=config.get("policies.role_name"),
session_name="create_role_" + username,
)
results = {"errors": 0, "role_created": "false", "action_results": []}
try:
await sync_to_async(iam_client.create_role)(
RoleName=create_model.role_name,
AssumeRolePolicyDocument=json.dumps(default_trust_policy),
Description=description,
Tags=[],
)
results["action_results"].append(
{
"status": "success",
"message": f"Role arn:aws:iam::{create_model.account_id}:role/{create_model.role_name} "
f"successfully created",
}
)
results["role_created"] = "true"
except Exception as e:
log_data["message"] = "Exception occurred creating role"
log_data["error"] = str(e)
log.error(log_data, exc_info=True)
results["action_results"].append(
{
"status": "error",
"message": f"Error creating role {create_model.role_name} in account {create_model.account_id}:"
+ str(e),
}
)
results["errors"] += 1
sentry_sdk.capture_exception()
# Since we were unable to create the role, no point continuing, just return
return results
# If here, role has been successfully created, add status updates for each action
results["action_results"].append(
{
"status": "success",
"message": "Successfully added default Assume Role Policy Document",
}
)
results["action_results"].append(
{
"status": "success",
"message": "Successfully added description: " + description,
}
)
# Create instance profile and attach if specified
if create_model.instance_profile:
try:
await sync_to_async(iam_client.create_instance_profile)(
InstanceProfileName=create_model.role_name
)
await sync_to_async(iam_client.add_role_to_instance_profile)(
InstanceProfileName=create_model.role_name,
RoleName=create_model.role_name,
)
results["action_results"].append(
{
"status": "success",
"message": f"Successfully added instance profile {create_model.role_name} to role "
f"{create_model.role_name}",
}
)
except Exception as e:
log_data[
"message"
] = "Exception occurred creating/attaching instance profile"
log_data["error"] = str(e)
log.error(log_data, exc_info=True)
sentry_sdk.capture_exception()
results["action_results"].append(
{
"status": "error",
"message": f"Error creating/attaching instance profile {create_model.role_name} to role: "
+ str(e),
}
)
results["errors"] += 1
stats.count(
f"{log_data['function']}.success", tags={"role_name": create_model.role_name}
)
log_data["message"] = "Successfully created role"
log.info(log_data)
return results
async def prepare(self):
self.tracer = None
self.span = None
self.spans = {}
self.responses = []
self.request_uuid = str(uuid.uuid4())
self.auth_cookie_expiration = 0
stats.timer("base_handler.incoming_request")
if config.get("auth.require_mtls", False):
try:
await auth.validate_certificate(self.request.headers)
except InvalidCertificateException:
stats.count(
"GetCredentialsHandler.post.invalid_certificate_header_value"
)
self.set_status(403)
self.write({"code": "403", "message": "Invalid Certificate"})
await self.finish()
return
# Extract user from valid certificate
try:
self.requester = await auth.extract_user_from_certificate(
self.request.headers)
self.current_cert_age = await auth.get_cert_age_seconds(
self.request.headers)
except (MissingCertificateException, Exception) as e:
if isinstance(e, MissingCertificateException):
stats.count(
"BaseMtlsHandler.post.missing_certificate_header")
message = "Missing Certificate in Header."
else:
stats.count("BaseMtlsHandler.post.exception")
message = f"Invalid Mtls Certificate: {e}"
self.set_status(400)
self.write({"code": "400", "message": message})
await self.finish()
return
elif config.get("auth.require_jwt", True):
# Check to see if user has a valid auth cookie
if config.get("auth_cookie_name", "consoleme_auth"):
auth_cookie = self.get_cookie(
config.get("auth_cookie_name", "consoleme_auth"))
if auth_cookie:
res = await validate_and_return_jwt_token(auth_cookie)
if not res:
error = {
"code": "invalid_jwt",
"message": "JWT is invalid or has expired.",
"request_id": self.request_uuid,
}
self.set_status(403)
self.write(error)
await self.finish()
self.user = res.get("user")
self.groups = res.get("groups")
self.requester = {"type": "user", "email": self.user}
self.current_cert_age = int(time.time()) - res.get("iat")
self.auth_cookie_expiration = res.get("exp")
else:
raise MissingConfigurationValue(
"Auth cookie name is not defined in configuration.")
else:
raise MissingConfigurationValue(
"Unsupported authentication scheme.")
if not hasattr(self, "requester"):
raise tornado.web.HTTPError(403, "Unable to authenticate user.")
self.ip = self.get_request_ip()
await self.configure_tracing()