async def retrieve_accounts_from_aws_organizations() -> CloudAccountModelArray: """ Polls AWS Organizations for our Account ID to Account Name mapping :param: null :return: CloudAccountModelArray """ cloud_accounts = [] for organization in config.get("cache_accounts_from_aws_organizations", []): organizations_master_account_id = organization.get( "organizations_master_account_id" ) role_to_assume = organization.get( "organizations_master_role_to_assume", config.get("policies.role_name"), ) if not organizations_master_account_id: raise MissingConfigurationValue( "Your AWS Organizations Master Account ID is not specified in configuration. " "Unable to sync accounts from " "AWS Organizations" ) if not role_to_assume: raise MissingConfigurationValue( "ConsoleMe doesn't know what role to assume to retrieve account information " "from AWS Organizations. please set the appropriate configuration value." ) client = await sync_to_async(boto3_cached_conn)( "organizations", account_number=organizations_master_account_id, assume_role=role_to_assume, session_name="ConsoleMeOrganizationsSync", ) paginator = await sync_to_async(client.get_paginator)("list_accounts") page_iterator = await sync_to_async(paginator.paginate)() accounts = [] for page in page_iterator: accounts.extend(page["Accounts"]) for account in accounts: status = account["Status"].lower() cloud_accounts.append( CloudAccountModel( id=account["Id"], name=account["Name"], email=account["Email"], status=status, type="aws", sync_enabled=True, # TODO: Check for tag to disable sync? ) ) return CloudAccountModelArray(accounts=cloud_accounts)
async def populate_oidc_config(): http_client = tornado.httpclient.AsyncHTTPClient() metadata_url = config.get("get_user_by_oidc_settings.metadata_url") if metadata_url: res = await http_client.fetch( metadata_url, method="GET", headers={ "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", }, ) oidc_config = json.loads(res.body) else: authorization_endpoint = config.get( "get_user_by_oidc_settings.authorization_endpoint") token_endpoint = config.get("get_user_by_oidc_settings.token_endpoint") jwks_uri = config.get("get_user_by_oidc_settings.jwks_uri") if not (authorization_endpoint or token_endpoint or jwks_uri): raise MissingConfigurationValue("Missing OIDC Configuration.") oidc_config = { "authorization_endpoint": authorization_endpoint, "token_endpoint": token_endpoint, "jwks_uri": jwks_uri, } client_id = config.get("oidc_secrets.client_id") client_secret = config.get("oidc_secrets.secret") if not (client_id or client_secret): raise MissingConfigurationValue("Missing OIDC Secrets") oidc_config["client_id"] = client_id oidc_config["client_secret"] = client_secret # Fetch jwks_uri for jwt validation res = await http_client.fetch( oidc_config["jwks_uri"], method="GET", headers={ "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", }, ) oidc_config["jwks_data"] = json.loads(res.body) oidc_config["jwt_keys"] = {} for k in oidc_config["jwks_data"]["keys"]: key_type = k["kty"] key_id = k["kid"] if key_type == "RSA": oidc_config["jwt_keys"][key_id] = RSAAlgorithm.from_jwk( json.dumps(k)) elif key_type == "EC": oidc_config["jwt_keys"][key_id] = ECAlgorithm.from_jwk( json.dumps(k)) return oidc_config
async def get(self, requested_challenge_token): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) challenge_j = red.hget( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, ) if not challenge_j: self.write({"status": "unknown"}) return challenge = json.loads(challenge_j) # Delete the token if it has expired current_time = int( datetime.utcnow().replace(tzinfo=pytz.UTC).timestamp()) if challenge.get("ttl", 0) < current_time: red.hdel( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, ) self.write({"status": "expired"}) return ip = self.get_request_ip() if ip != challenge.get("ip"): self.write({"status": "unauthorized"}) return # Generate a jwt if user authentication was successful if challenge.get("status") == "success": jwt_expiration = datetime.utcnow().replace( tzinfo=pytz.UTC) + timedelta( minutes=config.get("jwt.expiration_minutes", 60)) encoded_jwt = await generate_jwt_token(challenge.get("user"), challenge.get("groups"), exp=jwt_expiration) self.write({ "status": challenge["status"], "cookie_name": config.get("auth_cookie_name", "consoleme_auth"), "expiration": int(jwt_expiration.timestamp()), "encoded_jwt": encoded_jwt, "user": challenge["user"], }) # Delete the token so that it cannot be re-used red.hdel( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, ) return self.write({"status": challenge.get("status")}) return
async def is_object_older_than_seconds(key: str, older_than_seconds: int, bucket: Optional[str] = None, s3_client=None) -> bool: """ This function checks if an S3 object is older than the specified number of seconds. if the object doesn't exist, this function will return True. """ if not bucket: bucket = config.get("consoleme_s3_bucket") if not bucket: raise MissingConfigurationValue( "`bucket` not defined, and we can't find the default bucket in " "the configuration key `consoleme_s3_bucket`.") now = datetime.utcnow().replace(tzinfo=pytz.utc) if not s3_client: s3_client = boto3.client("s3", **config.get("boto3.client_kwargs", {})) try: res = await sync_to_async(s3_client.head_object)(Bucket=bucket, Key=key) except ClientError as e: # If file is not found, we'll tell the user it's older than the specified time if e.response.get("Error", {}).get("Code") == "404": return True raise datetime_value = res["LastModified"] if (now - datetime_value).total_seconds() > older_than_seconds: return True return False
async def _generate_inline_policy_statement_from_mapping( generator: ChangeGeneratorModel, ) -> Dict: """ Generates an inline policy statement given a ChangeGeneratorModel from a action mapping stored in configuration. :param generator: ChangeGeneratorModel :return: policy_statement: A dictionary representing an inline policy statement. """ generator_type = generator.generator_type if not isinstance(generator_type, str): generator_type = generator.generator_type.value permissions_map = config.get( f"self_service_iam.permissions_map.{generator_type}.action_map") if not permissions_map: raise MissingConfigurationValue( f"Unable to find applicable action map configuration for {generator_type}." ) action_group_actions: List[str] = [] resource_arns = [generator.resource_arn] effect = generator.effect for action in generator.action_groups: # TODO: Seems like a datamodel bug when we don't have a enum defined for an array type, but I need to access # this as a string sometimes if isinstance(action, str): action_group_actions.append(action) else: action_group_actions.append(action) actions = await _get_actions_from_groups(action_group_actions, permissions_map) condition: Optional[Dict] = await _generate_condition_with_substitutions( generator) return await _generate_policy_statement(actions, resource_arns, effect, condition)
async def _generate_inline_policy_statement_from_policy_sentry( generator: CrudChangeGeneratorModel, ) -> Dict: """ Generates an inline policy statement given a ChangeGeneratorModel from a action mapping provided by policy sentry. :param generator: ChangeGeneratorModel :return: policy_statement: A dictionary representing an inline policy statement. """ permissions_map = (self_service_iam_config.get("permissions_map", {}).get( "crud_lookup", {}).get("action_map")) if not permissions_map: raise MissingConfigurationValue( "Unable to find applicable action map configuration.") access_level_actions: List[str] = [] for access in generator.action_groups: for pm in permissions_map: if pm["name"] == access: access_level_actions += pm.get("permissions") actions = await _get_policy_sentry_access_level_actions( generator.service_name, access_level_actions) if generator.extra_actions: actions.extend(generator.extra_actions) if isinstance(generator.resource_arn, str): generator.resource_arn = [generator.resource_arn] return await _generate_policy_statement(actions, generator.resource_arn, generator.effect, generator.condition)
async def post(self, requested_challenge_token): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) data = tornado.escape.json_decode(self.request.body) ip = self.request.headers.get("X-Forwarded-For", self.request.remote_ip).split( "," ) if isinstance(ip, list): ip = ip[0] log_data = { "user": self.user, "function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}", "requested_challenge_token": requested_challenge_token, "message": "Incoming request", "ip": ip, } log.debug(log_data) all_challenges = red.hgetall( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP") ) if not all_challenges: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL." ) self.write({"message": message}) return await delete_expired_challenges(all_challenges) valid_user_challenge = await retrieve_user_challenge( self, requested_challenge_token, log_data ) if not valid_user_challenge: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL." ) self.write({"message": message}) return if data.get("nonce") != valid_user_challenge["nonce"]: message = "Unable to validate challenge URL. The Nonce you've submitted is invalid." log.error({**log_data, "message": message}) self.write({"message": message}) return valid_user_challenge["status"] = "success" valid_user_challenge["user"] = self.user valid_user_challenge["groups"] = self.groups red.hset( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, json.dumps(valid_user_challenge), ) message = "You've successfully authenticated to ConsoleMe and may now close this page." self.write({"message": message})
async def get(self, requested_challenge_token): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) log_data = { "user": self.user, "function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}", "requested_challenge_token": requested_challenge_token, "message": "Incoming request", "ip": self.ip, } log.debug(log_data) all_challenges = red.hgetall( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP")) if not all_challenges: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL.") self.write({"message": message}) return await delete_expired_challenges(all_challenges) valid_user_challenge = await retrieve_user_challenge( self, requested_challenge_token, log_data) if not valid_user_challenge: return if valid_user_challenge.get("visited"): message = ("This unique challenge URL has already been viewed. " "Please try requesting a new challenge URL.") self.write({"message": message}) return valid_user_challenge["visited"] = True valid_user_challenge["nonce"] = str(uuid.uuid4()) red.hset( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, json.dumps(valid_user_challenge), ) request_ip = valid_user_challenge["ip"] request_user = valid_user_challenge["user"] message = ( f"A user at **{request_ip}** has requested ConsoleMe credentials for **{request_user}**.\n\n" f"You must approve this request for credentials to be provided. " f"You will not be able to refresh or revisit this page after closing it.\n\n" f"If you did not create this request, please report it to your security team." ) self.write({ "message": message, "nonce": valid_user_challenge["nonce"], "show_approve_button": True, })
def query(query: str, use_aggregator: bool = True, account_id: Optional[str] = None) -> List: resources = [] if use_aggregator: config_client = boto3.client("config", region_name=config.region) configuration_aggregator_name: str = config.get( "aws_config.configuration_aggregator.name").format( region=config.region) if not configuration_aggregator_name: raise MissingConfigurationValue( "Invalid configuration for aws_config") response = config_client.select_aggregate_resource_config( Expression=query, ConfigurationAggregatorName=configuration_aggregator_name, Limit=100, ) for r in response.get("Results", []): resources.append(json.loads(r)) while response.get("NextToken"): response = config_client.select_aggregate_resource_config( Expression=query, ConfigurationAggregatorName=configuration_aggregator_name, Limit=100, NextToken=response["NextToken"], ) for r in response.get("Results", []): resources.append(json.loads(r)) return resources else: # Don't use Config aggregator and instead query all the regions on an account session = boto3.Session() available_regions = session.get_available_regions("config") excluded_regions = config.get( "api_protect.exclude_regions", ["af-south-1", "ap-east-1", "eu-south-1", "me-south-1"], ) regions = [x for x in available_regions if x not in excluded_regions] for region in regions: config_client = boto3_cached_conn( "config", account_number=account_id, assume_role=config.get("policies.role_name"), region=region, ) response = config_client.select_resource_config(Expression=query, Limit=100) for r in response.get("Results", []): resources.append(json.loads(r)) # Query Config for a specific account in all regions we care about while response.get("NextToken"): response = config_client.select_resource_config( Expression=query, Limit=100, NextToken=response["NextToken"]) for r in response.get("Results", []): resources.append(json.loads(r)) return resources
async def retrieve_accounts_from_swag() -> CloudAccountModelArray: function: str = f"{sys._getframe().f_code.co_name}" expected_owners: List = config.get( "retrieve_accounts_from_swag.expected_owners", []) swag_base_url = config.get("retrieve_accounts_from_swag.base_url") if not swag_base_url: raise MissingConfigurationValue( "Unable to find Swag URL in configuration") swag_url = swag_base_url + "api/1/accounts" try: http_client = AsyncHTTPClient(force_instance=True) resp = await http_client.fetch( swag_url, headers={ "Content-Type": "application/json", "Accept": "application/json" }, ) except (ConnectionError, HTTPClientError) as e: log.error( { "message": "Unable to connect to SWAG", "error": str(e), "function": function, }, exc_info=True, ) stats.count(f"{function}.connectionerror") raise swag_accounts = json.loads(resp.body) cloud_accounts = [] for account in swag_accounts: # Ignore third party accounts if expected_owners and account.get("owner") not in expected_owners: continue account_status = account["account_status"] sync_enabled = False if account_status == "ready": account_status = "active" sync_enabled = True cloud_accounts.append( CloudAccountModel( id=account["id"], name=account["name"], email=account["email"], status=account_status, sync_enabled=sync_enabled, sensitive=account["sensitive"], environment=account["environment"], aliases=account["aliases"], type="aws", )) return CloudAccountModelArray(accounts=cloud_accounts)
async def get_conglomo_url_for_resource( account_id, resource_id, technology, region="global" ): conglomo_url = config.get("get_aws_config_history_url_for_resource.conglomo_url") if not conglomo_url: raise MissingConfigurationValue( "Unable to find conglomo URL in configuration: `get_aws_config_history_url_for_resource.conglomo_url`" ) encoded_resource_id = base64.urlsafe_b64encode(resource_id.encode("utf-8")).decode( "utf-8" ) return f"{conglomo_url}/resource/{account_id}/{region}/{technology}/{encoded_resource_id}"
async def populate_oidc_config(): http_client = tornado.httpclient.AsyncHTTPClient() metadata_url = config.get( "get_user_by_aws_alb_auth_settings.access_token_validation.metadata_url" ) if metadata_url: res = await http_client.fetch( metadata_url, method="GET", headers={ "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", }, ) oidc_config = json.loads(res.body) else: jwks_uri = config.get( "get_user_by_aws_alb_auth_settings.access_token_validation.jwks_uri" ) if not jwks_uri: raise MissingConfigurationValue("Missing OIDC Configuration.") oidc_config = { "jwks_uri": jwks_uri, } # Fetch jwks_uri for jwt validation res = await http_client.fetch( oidc_config["jwks_uri"], method="GET", headers={ "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", }, ) oidc_config["jwks_data"] = json.loads(res.body) oidc_config["jwt_keys"] = {} for k in oidc_config["jwks_data"]["keys"]: key_type = k["kty"] key_id = k["kid"] if key_type == "RSA": oidc_config["jwt_keys"][key_id] = RSAAlgorithm.from_jwk( json.dumps(k)) elif key_type == "EC": oidc_config["jwt_keys"][key_id] = ECAlgorithm.from_jwk( json.dumps(k)) oidc_config["aud"] = config.get( "get_user_by_aws_alb_auth_settings.access_token_validation.client_id") return oidc_config
async def get(self, user): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) ip = self.request.headers.get("X-Forwarded-For", self.request.remote_ip).split(",") if isinstance(ip, list): ip = ip[0] token = str(uuid.uuid4()) entry = { "ttl": int((datetime.utcnow().replace(tzinfo=pytz.UTC) + timedelta(minutes=2)).timestamp()), "ip": ip, "status": "pending", "user": user, } red.hset( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), token, json.dumps(entry), ) challenge_url = "{url}/challenge_validator/{token}".format( url=config.get("url"), token=token) polling_url = "{url}/noauth/v1/challenge_poller/{token}".format( url=config.get("url"), token=token) self.write({ "challenge_url": challenge_url, "polling_url": polling_url }) log_data = { "function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}", "challenge_url": challenge_url, "polling_url": polling_url, "message": "Incoming request", "ip": ip, "user": user, } log.debug(log_data)
async def validate_and_return_api_caller(self, headers: dict): cli_auth_required_headers = config.get("cli_auth.required_headers") if not cli_auth_required_headers: raise MissingConfigurationValue( "You must specified the header key and expected value in order to validate a certificate for mutual " "TLS authentication. Refer to the `cli_auth.required_headers` configuration" ) for header in cli_auth_required_headers: for k, v in header.items(): if headers.get(k) != v: raise Exception( f"Header {k} is supposed to equal {v}, but it equals {headers.get(k)}." ) cert = await self.extract_user_from_certificate(headers) user = cert.get("name") if not user or user not in config.get("api_auth.valid_entities", []): raise Exception("Not authorized to call this API with that certificate.") return user
async def _generate_s3_inline_policy_statement_from_mapping( generator: ChangeGeneratorModel, ) -> Dict: """ Generates an S3 inline policy statement from a ChangeGeneratorModel. S3 is an edge case, thus it gets a unique function for this purpose. We need to consider the resource ARN and prefix. :param generator: ChangeGeneratorModel :return: policy_statement: A dictionary representing an inline policy statement. """ permissions_map = (self_service_iam_config.get("permissions_map", {}).get( "s3", {}).get("action_map")) if not permissions_map: raise MissingConfigurationValue( "Unable to find applicable action map configuration.") action_group_actions: List[str] = [] resource_arns = [] effect = generator.effect condition = generator.condition if isinstance(generator.resource_arn, str): generator.resource_arn = [generator.resource_arn] # Handle the bucket ARNs for arn in generator.resource_arn: if not arn.startswith("arn:aws:s3:::"): arn = f"arn:aws:s3:::{arn}" resource_arns.append(arn) # Make sure prefix starts with "/" if not generator.bucket_prefix.startswith("/"): generator.bucket_prefix = f"/{generator.bucket_prefix}" resource_arns.append(f"{arn}{generator.bucket_prefix}") for action in generator.action_groups: action_group_actions.append(action) actions = await _get_actions_from_groups(action_group_actions, permissions_map) if generator.extra_actions: actions.extend(generator.extra_actions) return await _generate_policy_statement(actions, resource_arns, effect, condition)
async def validate_certificate(self, headers: dict): cli_auth_required_headers = config.get("cli_auth.required_headers") if not cli_auth_required_headers: raise MissingConfigurationValue( "You must specified the header key and expected value in order to validate a certificate for mutual " "TLS authentication. Refer to the `cli_auth.required_headers` configuration" ) for header in cli_auth_required_headers: for k, v in header.items(): if headers.get(k) != v: stats.count("auth.validate_certificate.error") error = ( "Header {} is supposed to equal {}, but it equals {}.". format(k, v, headers.get(k))) log_data = { "function": "auth.validate_certificate", "message": error, } log.error(log_data) raise InvalidCertificateException(error) return True
async def get(self, requested_challenge_token): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) log_data = { "user": self.user, "function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}", "requested_challenge_token": requested_challenge_token, "message": "Incoming request", "ip": self.ip, } log.debug(log_data) all_challenges = red.hgetall( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP")) if not all_challenges: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL.") self.write({"message": message}) return await delete_expired_challenges(all_challenges) valid_user_challenge = await retrieve_user_challenge( self, requested_challenge_token, log_data) if not valid_user_challenge: return if valid_user_challenge.get("visited"): message = ("This unique challenge URL has already been viewed. " "Please try requesting a new challenge URL.") self.write({"message": message}) return request_ip = self.get_request_ip() # By default, the challenge URL requester IP must match the URL the challenge was created with. In some cases # (i.e. IPv4 vs IPv6), the challenge may have been created with an IPv4 address, and the authenticated browser # verification request may originate from an IPv6 one, or visa versa, in which case this configuration may # need to be explicitly set to False. if config.get( "challenge_url.request_ip_must_match_challenge_creation_ip", True): if request_ip != valid_user_challenge.get("ip"): log.error({ **log_data, "request_ip": request_ip, "challenge_ip": valid_user_challenge.get("ip"), "message": "Request IP doesn't match challenge IP", }) self.write({ "message": "Your originating IP doesn't match the IP the challenge was created with." }) return valid_user_challenge["visited"] = True valid_user_challenge["nonce"] = str(uuid.uuid4()) red.hset( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, json.dumps(valid_user_challenge), ) request_ip = valid_user_challenge["ip"] request_user = valid_user_challenge["user"] message = ( f"A user at **{request_ip}** has requested ConsoleMe credentials for **{request_user}**.\n\n" f"You must approve this request for credentials to be provided. " f"You will not be able to refresh or revisit this page after closing it.\n\n" f"If you did not create this request, please report it to your security team." ) self.write({ "message": message, "nonce": valid_user_challenge["nonce"], "show_approve_button": True, })
async def get_service(service_name: str, service_path: str, group: str) -> Resource: """Get service connection.""" function = f"{__name__}.{sys._getframe().f_code.co_name}" stats.count(function) log_data = { "function": function, "service_name": service_name, "service_path": service_path, "group": group, "message": f"Building service connection for {service_name} / {service_path}", } log.debug(log_data) if config.get("google.service_key_file"): admin_credentials = service_account.Credentials.from_service_account_file( config.get("google.service_key_file"), scopes=config.get( "google.admin_scopes", ["https://www.googleapis.com/auth/admin.directory.group"], ), ) elif config.get("google.service_key_dict"): admin_credentials = service_account.Credentials.from_service_account_info( config.get("google.service_key_dict"), scopes=config.get( "google.admin_scopes", ["https://www.googleapis.com/auth/admin.directory.group"], ), ) else: raise MissingConfigurationValue( "Missing configuration for Google. You must configure either `google.service_key_file` " "or `google.service_key_dict`.") # Change credential subject based on group domain credential_subjects = config.get("google.credential_subject") credential_subject = None for k, v in credential_subjects.items(): if k == group.split("@")[1]: credential_subject = v break if not credential_subject: raise NoCredentialSubjectException( "Error: Unable to find Google credential subject for domain {}. " "{}".format( group.split("@")[1], config.get("ses.support_reference", ""))) admin_delegated_credentials = admin_credentials.with_subject( credential_subject) service = await sync_to_async(googleapiclient.discovery.build )(service_name, service_path, credentials=admin_delegated_credentials) return service
def query( query: str, use_aggregator: bool = True, account_id: Optional[str] = None ) -> List: resources = [] if use_aggregator: config_client = boto3.client("config", region_name=config.region) configuration_aggregator_name: str = config.get( "aws_config.configuration_aggregator.name" ).format(region=config.region) if not configuration_aggregator_name: raise MissingConfigurationValue("Invalid configuration for aws_config") response = config_client.select_aggregate_resource_config( Expression=query, ConfigurationAggregatorName=configuration_aggregator_name, Limit=100, ) for r in response.get("Results", []): resources.append(json.loads(r)) while response.get("NextToken"): response = config_client.select_aggregate_resource_config( Expression=query, ConfigurationAggregatorName=configuration_aggregator_name, Limit=100, NextToken=response["NextToken"], ) for r in response.get("Results", []): resources.append(json.loads(r)) return resources else: # Don't use Config aggregator and instead query all the regions on an account session = boto3.Session() available_regions = session.get_available_regions("config") excluded_regions = config.get( "api_protect.exclude_regions", ["af-south-1", "ap-east-1", "ap-northeast-3", "eu-south-1", "me-south-1"], ) regions = [x for x in available_regions if x not in excluded_regions] for region in regions: config_client = boto3_cached_conn( "config", account_number=account_id, assume_role=config.get("policies.role_name"), region=region, sts_client_kwargs=dict( region_name=config.region, endpoint_url=f"https://sts.{config.region}.amazonaws.com", ), ) try: response = config_client.select_resource_config( Expression=query, Limit=100 ) for r in response.get("Results", []): resources.append(json.loads(r)) # Query Config for a specific account in all regions we care about while response.get("NextToken"): response = config_client.select_resource_config( Expression=query, Limit=100, NextToken=response["NextToken"] ) for r in response.get("Results", []): resources.append(json.loads(r)) except ClientError as e: log.error( { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "message": "Failed to query AWS Config", "query": query, "use_aggregator": use_aggregator, "account_id": account_id, "region": region, "error": str(e), }, exc_info=True, ) sentry_sdk.capture_exception() return resources
async def clone_iam_role(clone_model: CloneRoleRequestModel, username): """ Clones IAM role within same account or across account, always creating and attaching instance profile if one exists on the source role. ;param username: username of user requesting action ;:param clone_model: CloneRoleRequestModel, which has the following attributes: account_id: source role's account ID role_name: source role's name dest_account_id: destination role's account ID (may be same as account_id) dest_role_name: destination role's name clone_options: dict to indicate what to copy when cloning: assume_role_policy: bool default: False - uses default ConsoleMe AssumeRolePolicy tags: bool default: False - defaults to no tags copy_description: bool default: False - defaults to copying provided description or default description description: string default: "Role cloned via ConsoleMe by `username` from `arn:aws:iam::<account_id>:role/<role_name>` if copy_description is True, then description is ignored inline_policies: bool default: False - defaults to no inline policies managed_policies: bool default: False - defaults to no managed policies :return: results: - indicating the results of each action """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "message": "Attempting to clone role", "account_id": clone_model.account_id, "role_name": clone_model.role_name, "dest_account_id": clone_model.dest_account_id, "dest_role_name": clone_model.dest_role_name, "user": username, } log.info(log_data) role = await fetch_role_details(clone_model.account_id, clone_model.role_name) default_trust_policy = config.get("user_role_creator.default_trust_policy") trust_policy = ( role.assume_role_policy_document if clone_model.options.assume_role_policy else default_trust_policy ) if trust_policy is None: raise MissingConfigurationValue( "Missing Default Assume Role Policy Configuration" ) if ( clone_model.options.copy_description and role.description is not None and role.description != "" ): description = role.description elif ( clone_model.options.description is not None and clone_model.options.description != "" ): description = clone_model.options.description else: description = f"Role cloned via ConsoleMe by {username} from {role.arn}" tags = role.tags if clone_model.options.tags and role.tags else [] iam_client = await sync_to_async(boto3_cached_conn)( "iam", service_type="client", account_number=clone_model.dest_account_id, region=config.region, assume_role=config.get("policies.role_name"), session_name="clone_role_" + username, ) results = {"errors": 0, "role_created": "false", "action_results": []} try: await sync_to_async(iam_client.create_role)( RoleName=clone_model.dest_role_name, AssumeRolePolicyDocument=json.dumps(trust_policy), Description=description, Tags=tags, ) results["action_results"].append( { "status": "success", "message": f"Role arn:aws:iam::{clone_model.dest_account_id}:role/{clone_model.dest_role_name} " f"successfully created", } ) results["role_created"] = "true" except Exception as e: log_data["message"] = "Exception occurred creating cloned role" log_data["error"] = str(e) log.error(log_data, exc_info=True) results["action_results"].append( { "status": "error", "message": f"Error creating role {clone_model.dest_role_name} in account {clone_model.dest_account_id}:" + str(e), } ) results["errors"] += 1 sentry_sdk.capture_exception() # Since we were unable to create the role, no point continuing, just return return results if clone_model.options.tags: results["action_results"].append( {"status": "success", "message": "Successfully copied tags"} ) if clone_model.options.assume_role_policy: results["action_results"].append( { "status": "success", "message": "Successfully copied Assume Role Policy Document", } ) else: results["action_results"].append( { "status": "success", "message": "Successfully added default Assume Role Policy Document", } ) if ( clone_model.options.copy_description and role.description is not None and role.description != "" ): results["action_results"].append( {"status": "success", "message": "Successfully copied description"} ) elif clone_model.options.copy_description: results["action_results"].append( { "status": "error", "message": "Failed to copy description, so added default description: " + description, } ) else: results["action_results"].append( { "status": "success", "message": "Successfully added description: " + description, } ) # Create instance profile and attach if it exists in source role if len(list(await sync_to_async(role.instance_profiles.all)())) > 0: try: await sync_to_async(iam_client.create_instance_profile)( InstanceProfileName=clone_model.dest_role_name ) await sync_to_async(iam_client.add_role_to_instance_profile)( InstanceProfileName=clone_model.dest_role_name, RoleName=clone_model.dest_role_name, ) results["action_results"].append( { "status": "success", "message": f"Successfully added instance profile {clone_model.dest_role_name} to role " f"{clone_model.dest_role_name}", } ) except Exception as e: log_data[ "message" ] = "Exception occurred creating/attaching instance profile" log_data["error"] = str(e) log.error(log_data, exc_info=True) sentry_sdk.capture_exception() results["action_results"].append( { "status": "error", "message": f"Error creating/attaching instance profile {clone_model.dest_role_name} to role: " + str(e), } ) results["errors"] += 1 # other optional attributes to copy over after role has been successfully created cloned_role = await fetch_role_details( clone_model.dest_account_id, clone_model.dest_role_name ) # Copy inline policies if clone_model.options.inline_policies: for src_policy in await sync_to_async(role.policies.all)(): await sync_to_async(src_policy.load)() try: dest_policy = await sync_to_async(cloned_role.Policy)(src_policy.name) await sync_to_async(dest_policy.put)( PolicyDocument=json.dumps(src_policy.policy_document) ) results["action_results"].append( { "status": "success", "message": f"Successfully copied inline policy {src_policy.name}", } ) except Exception as e: log_data["message"] = "Exception occurred copying inline policy" log_data["error"] = str(e) log.error(log_data, exc_info=True) sentry_sdk.capture_exception() results["action_results"].append( { "status": "error", "message": f"Error copying inline policy {src_policy.name}: " + str(e), } ) results["errors"] += 1 # Copy managed policies if clone_model.options.managed_policies: for src_policy in await sync_to_async(role.attached_policies.all)(): await sync_to_async(src_policy.load)() dest_policy_arn = src_policy.arn.replace( clone_model.account_id, clone_model.dest_account_id ) try: await sync_to_async(cloned_role.attach_policy)( PolicyArn=dest_policy_arn ) results["action_results"].append( { "status": "success", "message": f"Successfully attached managed policy {src_policy.arn} as {dest_policy_arn}", } ) except Exception as e: log_data["message"] = "Exception occurred copying managed policy" log_data["error"] = str(e) log.error(log_data, exc_info=True) sentry_sdk.capture_exception() results["action_results"].append( { "status": "error", "message": f"Error attaching managed policy {dest_policy_arn}: " + str(e), } ) results["errors"] += 1 stats.count( f"{log_data['function']}.success", tags={"role_name": clone_model.role_name} ) log_data["message"] = "Successfully cloned role" log.info(log_data) return results
async def detect_cloudtrail_denies_and_update_cache(): log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"} dynamo = UserDynamoHandler() queue_arn = config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.queue_arn", "").format(region=config.region) if not queue_arn: raise MissingConfigurationValue( "Unable to find required configuration value: " "`event_bridge.detect_cloudtrail_denies_and_update_cache.queue_arn`" ) queue_name = queue_arn.split(":")[-1] queue_account_number = queue_arn.split(":")[4] queue_region = queue_arn.split(":")[3] # Optionally assume a role before receiving messages from the queue queue_assume_role = config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.assume_role") sqs_client = await sync_to_async(boto3_cached_conn)( "sqs", service_type="client", region=queue_region, retry_max_attempts=2, account_number=queue_account_number, assume_role=queue_assume_role, ) queue_url_res = await sync_to_async(sqs_client.get_queue_url )(QueueName=queue_name) queue_url = queue_url_res.get("QueueUrl") if not queue_url: raise DataNotRetrievable( f"Unable to retrieve Queue URL for {queue_arn}") ct_events = [] messages_awaitable = await sync_to_async(sqs_client.receive_message )(QueueUrl=queue_url, MaxNumberOfMessages=10) messages = messages_awaitable.get("Messages", []) while messages: processed_messages = [] for message in messages: try: message_body = json.loads(message["Body"]) decoded_message = json.loads(message_body["Message"])["detail"] event_name = decoded_message.get("eventName") event_source = decoded_message.get("eventSource") for event_source_substitution in config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.event_bridge_substitutions", [".amazonaws.com"], ): event_source = event_source.replace( event_source_substitution, "") event_time = decoded_message.get("eventTime") utc_time = datetime.strptime(event_time, "%Y-%m-%dT%H:%M:%SZ") epoch_event_time = int( (utc_time - datetime(1970, 1, 1)).total_seconds() * 1000) try: session_name = decoded_message["userIdentity"][ "arn"].split("/")[-1] except ( IndexError, KeyError, ): # If IAM user, there won't be a session name session_name = "" try: role_arn = decoded_message["userIdentity"][ "sessionContext"]["sessionIssuer"]["arn"] except KeyError: # Skip events without a parsable ARN continue ct_event = dict( error_code=decoded_message.get("errorCode"), error_message=decoded_message.get("errorMessage"), arn=role_arn, session_name=session_name, request_id=decoded_message["requestID"], event_call=f"{event_source}:{event_name}", epoch_event_time=epoch_event_time, ttl=(epoch_event_time + 86400000) / 1000, ) ct_event["resource"] = await get_resource_from_cloudtrail_deny( ct_event) generated_policy = await generate_policy_from_cloudtrail_deny( ct_event) if generated_policy: ct_event["generated_policy"] = generated_policy ct_events.append(ct_event) except Exception as e: log.error({**log_data, "error": str(e)}, exc_info=True) sentry_sdk.capture_exception() processed_messages.append({ "Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"], }) await sync_to_async(sqs_client.delete_message_batch )(QueueUrl=queue_url, Entries=processed_messages) await sync_to_async(dynamo.batch_write_cloudtrail_events)(ct_events) messages_awaitable = await sync_to_async(sqs_client.receive_message )(QueueUrl=queue_url, MaxNumberOfMessages=10) messages = messages_awaitable.get("Messages", []) log.debug({ **log_data, "num_events": len(ct_events), "message": "Successfully cached Cloudtrail Access Denies", }) return ct_events
async def detect_cloudtrail_denies_and_update_cache( celery_app, event_ttl=config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.event_ttl", 86400), max_num_messages_to_process=config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.max_num_messages_to_process", 100, ), ) -> Dict[str, Any]: log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"} dynamo = UserDynamoHandler() queue_arn = config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.queue_arn", "").format(region=config.region) if not queue_arn: raise MissingConfigurationValue( "Unable to find required configuration value: " "`event_bridge.detect_cloudtrail_denies_and_update_cache.queue_arn`" ) queue_name = queue_arn.split(":")[-1] queue_account_number = queue_arn.split(":")[4] queue_region = queue_arn.split(":")[3] # Optionally assume a role before receiving messages from the queue queue_assume_role = config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.assume_role") # Modify existing cloudtrail deny samples all_cloudtrail_denies_l = await dynamo.parallel_scan_table_async( dynamo.cloudtrail_table) all_cloudtrail_denies = {} for cloudtrail_deny in all_cloudtrail_denies_l: all_cloudtrail_denies[cloudtrail_deny["request_id"]] = cloudtrail_deny sqs_client = await sync_to_async(boto3_cached_conn)( "sqs", service_type="client", region=queue_region, retry_max_attempts=2, account_number=queue_account_number, assume_role=queue_assume_role, client_kwargs=config.get("boto3.client_kwargs", {}), ) queue_url_res = await sync_to_async(sqs_client.get_queue_url )(QueueName=queue_name) queue_url = queue_url_res.get("QueueUrl") if not queue_url: raise DataNotRetrievable( f"Unable to retrieve Queue URL for {queue_arn}") messages_awaitable = await sync_to_async(sqs_client.receive_message )(QueueUrl=queue_url, MaxNumberOfMessages=10) new_events = 0 messages = messages_awaitable.get("Messages", []) num_events = 0 reached_limit_on_num_messages_to_process = False while messages: if num_events >= max_num_messages_to_process: reached_limit_on_num_messages_to_process = True break processed_messages = [] for message in messages: try: message_body = json.loads(message["Body"]) try: if "Message" in message_body: decoded_message = json.loads( message_body["Message"])["detail"] else: decoded_message = message_body["detail"] except Exception as e: log.error({ **log_data, "message": "Unable to process Cloudtrail message", "message_body": message_body, "error": str(e), }) sentry_sdk.capture_exception() continue event_name = decoded_message.get("eventName") event_source = decoded_message.get("eventSource") for event_source_substitution in config.get( "event_bridge.detect_cloudtrail_denies_and_update_cache.event_bridge_substitutions", [".amazonaws.com"], ): event_source = event_source.replace( event_source_substitution, "") event_time = decoded_message.get("eventTime") utc_time = datetime.strptime(event_time, "%Y-%m-%dT%H:%M:%SZ") epoch_event_time = int( (utc_time - datetime(1970, 1, 1)).total_seconds()) # Skip entries older than a day if int(time.time()) - 86400 > epoch_event_time: continue try: session_name = decoded_message["userIdentity"][ "arn"].split("/")[-1] except ( IndexError, KeyError, ): # If IAM user, there won't be a session name session_name = "" try: principal_arn = decoded_message["userIdentity"][ "sessionContext"]["sessionIssuer"]["arn"] except KeyError: # Skip events without a parsable ARN continue event_call = f"{event_source}:{event_name}" ct_event = dict( error_code=decoded_message.get("errorCode"), error_message=decoded_message.get("errorMessage"), arn=principal_arn, # principal_owner=owner, session_name=session_name, source_ip=decoded_message["sourceIPAddress"], event_call=event_call, epoch_event_time=epoch_event_time, ttl=epoch_event_time + event_ttl, count=1, ) resource = await get_resource_from_cloudtrail_deny( ct_event, decoded_message) ct_event["resource"] = resource request_id = f"{principal_arn}-{session_name}-{event_call}-{resource}" ct_event["request_id"] = request_id generated_policy = await generate_policy_from_cloudtrail_deny( ct_event) if generated_policy: ct_event["generated_policy"] = generated_policy if all_cloudtrail_denies.get(request_id): existing_count = all_cloudtrail_denies[request_id].get( "count", 1) ct_event["count"] += existing_count all_cloudtrail_denies[request_id] = ct_event else: all_cloudtrail_denies[request_id] = ct_event new_events += 1 num_events += 1 except Exception as e: log.error({**log_data, "error": str(e)}, exc_info=True) sentry_sdk.capture_exception() processed_messages.append({ "Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"], }) if processed_messages: await sync_to_async(sqs_client.delete_message_batch )(QueueUrl=queue_url, Entries=processed_messages) await sync_to_async(dynamo.batch_write_cloudtrail_events )(all_cloudtrail_denies.values()) messages_awaitable = await sync_to_async(sqs_client.receive_message )(QueueUrl=queue_url, MaxNumberOfMessages=10) messages = messages_awaitable.get("Messages", []) if reached_limit_on_num_messages_to_process: # We hit our limit. Let's spawn another task immediately to process remaining messages celery_app.send_task( "consoleme.celery_tasks.celery_tasks.cache_cloudtrail_denies", ) log_data["message"] = "Successfully cached Cloudtrail Access Denies" log_data["num_events"] = num_events log_data["new_events"] = new_events log.debug(log_data) return log_data
async def create_iam_role(create_model: RoleCreationRequestModel, username): """ Creates IAM role. :param create_model: RoleCreationRequestModel, which has the following attributes: account_id: destination account's ID role_name: destination role name description: optional string - description of the role default: Role created by {username} through ConsoleMe instance_profile: optional boolean - whether to create an instance profile and attach it to the role or not default: True :param username: username of user requesting action :return: results: - indicating the results of each action """ log_data = { "function": f"{__name__}.{sys._getframe().f_code.co_name}", "message": "Attempting to create role", "account_id": create_model.account_id, "role_name": create_model.role_name, "user": username, } log.info(log_data) default_trust_policy = config.get("user_role_creator.default_trust_policy") if default_trust_policy is None: raise MissingConfigurationValue( "Missing Default Assume Role Policy Configuration" ) if create_model.description: description = create_model.description else: description = f"Role created by {username} through ConsoleMe" iam_client = await sync_to_async(boto3_cached_conn)( "iam", service_type="client", account_number=create_model.account_id, region=config.region, assume_role=config.get("policies.role_name"), session_name="create_role_" + username, ) results = {"errors": 0, "role_created": "false", "action_results": []} try: await sync_to_async(iam_client.create_role)( RoleName=create_model.role_name, AssumeRolePolicyDocument=json.dumps(default_trust_policy), Description=description, Tags=[], ) results["action_results"].append( { "status": "success", "message": f"Role arn:aws:iam::{create_model.account_id}:role/{create_model.role_name} " f"successfully created", } ) results["role_created"] = "true" except Exception as e: log_data["message"] = "Exception occurred creating role" log_data["error"] = str(e) log.error(log_data, exc_info=True) results["action_results"].append( { "status": "error", "message": f"Error creating role {create_model.role_name} in account {create_model.account_id}:" + str(e), } ) results["errors"] += 1 sentry_sdk.capture_exception() # Since we were unable to create the role, no point continuing, just return return results # If here, role has been successfully created, add status updates for each action results["action_results"].append( { "status": "success", "message": "Successfully added default Assume Role Policy Document", } ) results["action_results"].append( { "status": "success", "message": "Successfully added description: " + description, } ) # Create instance profile and attach if specified if create_model.instance_profile: try: await sync_to_async(iam_client.create_instance_profile)( InstanceProfileName=create_model.role_name ) await sync_to_async(iam_client.add_role_to_instance_profile)( InstanceProfileName=create_model.role_name, RoleName=create_model.role_name, ) results["action_results"].append( { "status": "success", "message": f"Successfully added instance profile {create_model.role_name} to role " f"{create_model.role_name}", } ) except Exception as e: log_data[ "message" ] = "Exception occurred creating/attaching instance profile" log_data["error"] = str(e) log.error(log_data, exc_info=True) sentry_sdk.capture_exception() results["action_results"].append( { "status": "error", "message": f"Error creating/attaching instance profile {create_model.role_name} to role: " + str(e), } ) results["errors"] += 1 stats.count( f"{log_data['function']}.success", tags={"role_name": create_model.role_name} ) log_data["message"] = "Successfully created role" log.info(log_data) return results
async def get_service(service_name: str, service_path: str, group: str) -> Resource: """ Get a service connection to Google. You'll need to generate a GCP service account first from instructions here: https://hawkins.gitbook.io/consoleme/configuration/authentication-and-authorization/google-groups-support ConsoleMe requires that you either have a service key file with content like below, and you've set the configuration for `google.service_key_file` to the full path of that file on disk, or you've just put the json for this in your ConsoleMe configuration in the `google.service_key_dict` configuration key. There are sensitive secrets here, so if you want to reference them directly in configuration, we encourage you to store these secrets in AWS Secrets Manager https://hawkins.gitbook.io/consoleme/configuration/aws-secret-manager-integration { "type": "service_account", "project_id": "cons...", "private_key_id": "dc61.....", "private_key": "-----BEGIN PRIVATE KEY-----\nMII.....", "client_email": "*****@*****.**", "client_id": "1234....", "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/consolem...." } """ function = f"{__name__}.{sys._getframe().f_code.co_name}" stats.count(function) log_data = { "function": function, "service_name": service_name, "service_path": service_path, "group": group, "message": f"Building service connection for {service_name} / {service_path}", } log.debug(log_data) if config.get("google.service_key_file"): admin_credentials = service_account.Credentials.from_service_account_file( config.get("google.service_key_file"), scopes=config.get( "google.admin_scopes", ["https://www.googleapis.com/auth/admin.directory.group"], ), ) elif config.get("google.service_key_dict"): admin_credentials = service_account.Credentials.from_service_account_info( config.get("google.service_key_dict"), scopes=config.get( "google.admin_scopes", ["https://www.googleapis.com/auth/admin.directory.group"], ), ) else: raise MissingConfigurationValue( "Missing configuration for Google. You must configure either `google.service_key_file` " "or `google.service_key_dict`.") # Change credential subject based on group domain credential_subjects = config.get("google.credential_subject") credential_subject = None for k, v in credential_subjects.items(): if k == group.split("@")[1]: credential_subject = v break if not credential_subject: raise NoCredentialSubjectException( "Error: Unable to find Google credential subject for domain {}. " "{}".format( group.split("@")[1], config.get("ses.support_reference", ""))) admin_delegated_credentials = admin_credentials.with_subject( credential_subject) service = await sync_to_async(googleapiclient.discovery.build )(service_name, service_path, credentials=admin_delegated_credentials) return service
def detect_role_changes_and_update_cache(celery_app): """ This function detects role changes through event bridge rules, and forces a refresh of the roles. """ log_data = {"function": f"{__name__}.{sys._getframe().f_code.co_name}"} queue_arn = config.get( "event_bridge.detect_role_changes_and_update_cache.queue_arn", "" ).format(region=config.region) if not queue_arn: raise MissingConfigurationValue( "Unable to find required configuration value: " "`event_bridge.detect_role_changes_and_update_cache.queue_arn`" ) queue_name = queue_arn.split(":")[-1] queue_account_number = queue_arn.split(":")[4] queue_region = queue_arn.split(":")[3] # Optionally assume a role before receiving messages from the queue queue_assume_role = config.get( "event_bridge.detect_role_changes_and_update_cache.assume_role" ) sqs_client = boto3_cached_conn( "sqs", service_type="client", region=queue_region, retry_max_attempts=2, account_number=queue_account_number, assume_role=queue_assume_role, ) queue_url_res = sqs_client.get_queue_url(QueueName=queue_name) queue_url = queue_url_res.get("QueueUrl") if not queue_url: raise DataNotRetrievable(f"Unable to retrieve Queue URL for {queue_arn}") roles_to_update = set() messages = sqs_client.receive_message( QueueUrl=queue_url, MaxNumberOfMessages=10 ).get("Messages", []) while messages: processed_messages = [] for message in messages: try: message_body = json.loads(message["Body"]) decoded_message = json.loads(message_body["Message"]) role_name = decoded_message["detail"]["requestParameters"]["roleName"] role_account_id = decoded_message["account"] role_arn = f"arn:aws:iam::{role_account_id}:role/{role_name}" if role_arn not in roles_to_update: celery_app.send_task( "consoleme.celery_tasks.celery_tasks.refresh_iam_role", args=[role_arn], ) roles_to_update.add(role_arn) except Exception as e: log.error( {**log_data, "error": str(e), "raw_message": message}, exc_info=True ) sentry_sdk.capture_exception() processed_messages.append( { "Id": message["MessageId"], "ReceiptHandle": message["ReceiptHandle"], } ) sqs_client.delete_message_batch(QueueUrl=queue_url, Entries=processed_messages) messages = sqs_client.receive_message( QueueUrl=queue_url, MaxNumberOfMessages=10 ).get("Messages", []) log.debug( { **log_data, "num_roles": len(roles_to_update), "message": "Triggered role cache update for roles that were created or changed", } ) return roles_to_update
async def prepare(self): self.tracer = None self.span = None self.spans = {} self.responses = [] self.request_uuid = str(uuid.uuid4()) self.auth_cookie_expiration = 0 stats.timer("base_handler.incoming_request") if config.get("auth.require_mtls", False): try: await auth.validate_certificate(self.request.headers) except InvalidCertificateException: stats.count( "GetCredentialsHandler.post.invalid_certificate_header_value" ) self.set_status(403) self.write({"code": "403", "message": "Invalid Certificate"}) await self.finish() return # Extract user from valid certificate try: self.requester = await auth.extract_user_from_certificate( self.request.headers) self.current_cert_age = await auth.get_cert_age_seconds( self.request.headers) except (MissingCertificateException, Exception) as e: if isinstance(e, MissingCertificateException): stats.count( "BaseMtlsHandler.post.missing_certificate_header") message = "Missing Certificate in Header." else: stats.count("BaseMtlsHandler.post.exception") message = f"Invalid Mtls Certificate: {e}" self.set_status(400) self.write({"code": "400", "message": message}) await self.finish() return elif config.get("auth.require_jwt", True): # Check to see if user has a valid auth cookie if config.get("auth_cookie_name", "consoleme_auth"): auth_cookie = self.get_cookie( config.get("auth_cookie_name", "consoleme_auth")) if auth_cookie: res = await validate_and_return_jwt_token(auth_cookie) if not res: error = { "code": "invalid_jwt", "message": "JWT is invalid or has expired.", "request_id": self.request_uuid, } self.set_status(403) self.write(error) await self.finish() self.user = res.get("user") self.groups = res.get("groups") self.requester = {"type": "user", "email": self.user} self.current_cert_age = int(time.time()) - res.get("iat") self.auth_cookie_expiration = res.get("exp") else: raise MissingConfigurationValue( "Auth cookie name is not defined in configuration.") else: raise MissingConfigurationValue( "Unsupported authentication scheme.") if not hasattr(self, "requester"): raise tornado.web.HTTPError(403, "Unable to authenticate user.") self.ip = self.get_request_ip() await self.configure_tracing()
async def get(self, requested_challenge_token): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) endpoint = self.kwargs.get("type") ip = self.request.headers.get("X-Forwarded-For", self.request.remote_ip).split(",") if isinstance(ip, list): ip = ip[0] log_data = { "user": self.user, "function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}", "requested_challenge_token": requested_challenge_token, "message": "Incoming request", "ip": ip, } log.debug(log_data) all_challenges = red.hgetall( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP")) current_time = int( datetime.utcnow().replace(tzinfo=pytz.UTC).timestamp()) expired_challenge_tokens = [] # Delete expired tokens if all_challenges: for token, challenge_j in all_challenges.items(): challenge = json.loads(challenge_j) if challenge.get("ttl", 0) < current_time: expired_challenge_tokens.append(token) if expired_challenge_tokens: red.hdel( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), *expired_challenge_tokens, ) else: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL.") if endpoint == "web": self.write(message) elif endpoint == "api": self.write({"message": message}) return # Get fresh challenge for user's request user_challenge_j = red.hget( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, ) if user_challenge_j: # Do a double-take check on the ttl # Delete the token user_challenge = json.loads(user_challenge_j) if user_challenge.get("ttl", 0) < current_time: message = "This challenge URL has expired. Please try requesting a new challenge URL." if endpoint == "web": self.write(message) elif endpoint == "api": self.write({"message": message}) return if ip != user_challenge.get("ip"): # Todo: Sometimes the request from the CLI will be IPv6, and the request from browser is ipv4. How # can we reconcile this so that we can perform validation? pass # raise Exception("IP address used to generate challenge URL is different than IP you are using now.") if self.user != user_challenge.get("user"): log_data = { **log_data, "message": "Authenticated user is different then user that requested token", "authenticated_user": self.user, "challenge_user": user_challenge.get("user"), } log.error(log_data) message = ( "This challenge URL is associated with a different user. Ensure that your client" "configuration is specifying the correct user.") if endpoint == "web": self.write(message) elif endpoint == "api": self.write({"message": message}) return user_challenge["status"] = "success" user_challenge["user"] = self.user user_challenge["groups"] = self.groups red.hset( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, json.dumps(user_challenge), ) message = "You've successfully authenticated to ConsoleMe and may now close this page." if endpoint == "web": self.write(message) elif endpoint == "api": self.write({"message": message}) else: message = "The requested challenge URL was not found. Please try requesting a new challenge URL." if endpoint == "web": self.write(message) elif endpoint == "api": self.write({"message": message}) return
async def post(self, requested_challenge_token): if not config.get("challenge_url.enabled", False): raise MissingConfigurationValue( "Challenge URL Authentication is not enabled in ConsoleMe's configuration" ) data = tornado.escape.json_decode(self.request.body) log_data = { "user": self.user, "function": f"{__name__}.{self.__class__.__name__}.{sys._getframe().f_code.co_name}", "requested_challenge_token": requested_challenge_token, "message": "Incoming request", "ip": self.ip, } log.debug(log_data) all_challenges = red.hgetall( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP")) if not all_challenges: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL.") self.write({"message": message}) return await delete_expired_challenges(all_challenges) valid_user_challenge = await retrieve_user_challenge( self, requested_challenge_token, log_data) if not valid_user_challenge: message = ( "Unable to find a matching challenge URL. This usually means that it has expired. " "Please try requesting a new challenge URL.") self.write({"message": message}) return if data.get("nonce") != valid_user_challenge["nonce"]: message = "Unable to validate challenge URL. The Nonce you've submitted is invalid." log.error({**log_data, "message": message}) self.write({"message": message}) return request_ip = self.get_request_ip() # By default, the challenge URL requester IP must match the URL the challenge was created with. In some cases # (i.e. IPv4 vs IPv6), the challenge may have been created with an IPv4 address, and the authenticated browser # verification request may originate from an IPv6 one, or visa versa, in which case this configuration may # need to be explicitly set to False. if config.get( "challenge_url.request_ip_must_match_challenge_creation_ip", True): if request_ip != valid_user_challenge.get("ip"): log.error({ **log_data, "request_ip": request_ip, "challenge_ip": valid_user_challenge.get("ip"), "message": "Request IP doesn't match challenge IP", }) self.write({ "message": "Your originating IP doesn't match the IP the challenge was created with." }) return valid_user_challenge["status"] = "success" valid_user_challenge["user"] = self.user valid_user_challenge["groups"] = self.groups red.hset( config.get("challenge_url.redis_key", "TOKEN_CHALLENGES_TEMP"), requested_challenge_token, json.dumps(valid_user_challenge), ) message = "You've successfully authenticated to ConsoleMe and may now close this page." self.write({"message": message})