def get_role(role, output='camelized', **conn): """ Orchestrates all the calls required to fully build out an IAM Role in the following format: { "Arn": ..., "AssumeRolePolicyDocument": ..., "CreateDate": ..., # str "InlinePolicies": ..., "InstanceProfiles": ..., "ManagedPolicies": ..., "Path": ..., "RoleId": ..., "RoleName": ..., } :param role: dict containing (at the very least) role_name and/or arn. :param output: Determines whether keys should be returned camelized or underscored. :param conn: dict containing enough information to make a connection to the desired account. Must at least have 'assume_role' key. :return: dict containing a fully built out role. """ role = modify(role, 'camelized') _conn_from_args(role, conn) role = _get_base(role, **conn) role.update( { 'managed_policies': get_role_managed_policies(role, **conn), 'inline_policies': get_role_inline_policies(role, **conn), 'instance_profiles': get_role_instance_profiles(role, **conn), '_version': 1 } ) return modify(role, format=output)
def _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source, add_no_repo=True): """ Perform a scaled down version of role update, this is used to get an accurate count of repoable permissions after a rollback or repo. Does update: - Policies - Aardvark data - Total permissions - Repoable permissions - Repoable services - Stats Does not update: - Filters - Active/inactive roles Args: role (Role) dynamo_table account_number conn (dict) source: repo, rollback, etc add_no_repo: if set to True newly discovered permissions will be added to no repo list Returns: None """ current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.update_role_data(dynamo_table, account_number, role, current_policies, source=source, add_no_repo=add_no_repo) aardvark_data = _get_aardvark_data(config['aardvark_api_location'], arn=role.arn) if not aardvark_data: return role.aa_data = aardvark_data[role.arn] roledata._calculate_repo_scores( [role], config['filter_config']['AgeFilter']['minimum_age'], hooks) set_role_data( dynamo_table, role.role_id, { 'AAData': role.aa_data, 'TotalPermissions': role.total_permissions, 'RepoablePermissions': role.repoable_permissions, 'RepoableServices': role.repoable_services }) roledata.update_stats(dynamo_table, [role], source=source)
def _fetch(self, arn: str) -> IAMEntry: # TODO: sort out arn vs role_id here # we probably only have role_id, which isn't sufficient for this implementation logger.info("getting role data for role %s", arn) conn = copy.deepcopy(self.config["connection_iam"]) conn["account_number"] = arn.split(":")[4] role = {"RoleName": arn.split("/")[-1]} role_policies: Dict[str, Any] = get_role_inline_policies(role, **conn) if not role_policies: raise NotFoundError self._data[arn] = role_policies return role_policies
def _get_iam_role_sync(account_id, role_name, conn) -> Optional[Dict[str, Any]]: client = boto3_cached_conn( "iam", account_number=account_id, assume_role=config.get("policies.role_name"), read_only=True, retry_max_attempts=2, ) role = client.get_role(RoleName=role_name)["Role"] role["ManagedPolicies"] = get_role_managed_policies( {"RoleName": role_name}, **conn) role["InlinePolicies"] = get_role_inline_policies( {"RoleName": role_name}, **conn) role["Tags"] = list_role_tags({"RoleName": role_name}, **conn) return role
def update_role_cache(account_number): conn = CONFIG['connection_iam'] conn['account_number'] = account_number roles = list_roles(**conn) active_roles = [] LOGGER.info('Updating role data for account {}'.format(account_number)) for role in tqdm(roles): role['policies'] = get_role_inline_policies(role, **conn) or {} active_roles.append(role['RoleId']) roledata.update_role_data(role) LOGGER.info('Finding inactive accounts') roledata.find_and_mark_inactive(active_roles) LOGGER.info('Filtering roles') filtered_roles_list = {} plugins = FilterPlugins() # need to have all roles in the dictionary, even if they aren't filtered filtered_roles_list = {role['RoleId']: [] for role in roles} for plugin in CONFIG.get('active_filters'): plugins.load_plugin(plugin) for plugin in plugins.filter_plugins: filtered_list = plugin.apply(roles) class_name = plugin.__class__.__name__ for role in filtered_list: LOGGER.info('Role {} filtered by {}'.format( role['RoleName'], class_name)) filtered_roles_list[role['RoleId']].append(class_name) roledata.update_filtered_roles(filtered_roles_list) LOGGER.info('Getting data from Aardvark') aardvark_data = _get_aardvark_data(account_number) LOGGER.info('Updating with Aardvark data') roledata.update_aardvark_data(account_number, aardvark_data) LOGGER.info('Calculating repoable permissions and services') roledata.update_repoable_data(_calculate_repo_scores(account_number)) LOGGER.info('Updating stats') roledata.update_stats(source='Scan')
def _rollback_role(account_number, role_name, dynamo_table, config, hooks, selection=None, commit=None): """ Display the historical policy versions for a roll as a numbered list. Restore to a specific version if selected. Indicate changes that will be made and then actually make them if commit is selected. Args: account_number (string) role_name (string) selection (int): which policy version in the list to rollback to commit (bool): actually make the change Returns: errors (list): if any """ errors = [] role_id = find_role_in_cache(dynamo_table, account_number, role_name) if not role_id: message = "Could not find role with name {} in account {}".format( role_name, account_number) errors.append(message) LOGGER.warning(message) return errors else: role = Role(get_role_data(dynamo_table, role_id)) # no option selected, display a table of options if not selection: headers = ["Number", "Source", "Discovered", "Permissions", "Services"] rows = [] for index, policies_version in enumerate(role.policies): policy_permissions, _ = roledata._get_permissions_in_policy( policies_version["Policy"]) rows.append([ index, policies_version["Source"], policies_version["Discovered"], len(policy_permissions), roledata._get_services_in_permissions(policy_permissions), ]) print(tabulate(rows, headers=headers)) return conn = config["connection_iam"] conn["account_number"] = account_number current_policies = get_role_inline_policies(role.as_dict(), **conn) if selection: pp = pprint.PrettyPrinter() print("Will restore the following policies:") pp.pprint(role.policies[int(selection)]["Policy"]) print("Current policies:") pp.pprint(current_policies) current_permissions, _ = roledata._get_permissions_in_policy( role.policies[-1]["Policy"]) selected_permissions, _ = roledata._get_permissions_in_policy( role.policies[int(selection)]["Policy"]) restored_permissions = selected_permissions - current_permissions print("\nResore will return these permissions:") print("\n".join([perm for perm in sorted(restored_permissions)])) if not commit: return False # if we're restoring from a version with fewer policies than we have now, we need to remove them to # complete the restore. To do so we'll store all the policy names we currently have and remove them # from the list as we update. Any policy names left need to be manually removed policies_to_remove = current_policies.keys() for policy_name, policy in role.policies[int(selection)]["Policy"].items(): try: LOGGER.info( f"Pushing cached policy: {policy_name} (role: {role.role_name} account {account_number})" ) put_role_policy( RoleName=role.role_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), **conn, ) except botocore.exceptions.ClientError as e: message = "Unable to push policy {}. Error: {} (role: {} account {})".format( policy_name, e.message, role.role_name, account_number) LOGGER.error(message, exc_info=True) errors.append(message) else: # remove the policy name if it's in the list try: policies_to_remove.remove(policy_name) except Exception: # nosec pass if policies_to_remove: for policy_name in policies_to_remove: try: LOGGER.info( f"Deleting policy {policy_name} for rollback (role: {role.role_name} account {account_number})" ) delete_role_policy(RoleName=role.role_name, PolicyName=policy_name, **conn) except botocore.excpetions.ClientError as e: message = "Unable to delete policy {}. Error: {} (role: {} account {})".format( policy_name, e.message, role.role_name, account_number) LOGGER.error(message, exc_info=True) errors.append(message) partial_update_role_data( role, dynamo_table, account_number, config, conn, hooks, source="Restore", add_no_repo=False, ) if not errors: LOGGER.info( f"Successfully restored selected version {selection} of role policies (role: {role.role_name} " f"account: {account_number}") return errors
def repo_role(account_number, role_name, dynamo_table, config, hooks, commit=False, scheduled=False): """ Calculate what repoing can be done for a role and then actually do it if commit is set 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data 2) Get the role's current permissions, repoable permissions, and the new policy if it will change 3) Make the changes if commit is set Args: account_number (string) role_name (string) commit (bool) Returns: None """ errors = [] role_id = find_role_in_cache(dynamo_table, account_number, role_name) # only load partial data that we need to determine if we should keep going role_data = get_role_data(dynamo_table, role_id, fields=['DisqualifiedBy', 'AAData', 'RepoablePermissions', 'RoleName']) if not role_data: LOGGER.warn('Could not find role with name {}'.format(role_name)) return else: role = Role(role_data) if len(role.disqualified_by) > 0: LOGGER.info('Cannot repo role {} in account {} because it is being disqualified by: {}'.format( role_name, account_number, role.disqualified_by)) return if not role.aa_data: LOGGER.warning('ARN not found in Access Advisor: {}'.format(role.arn)) return if not role.repoable_permissions: LOGGER.info('No permissions to repo for role {} in account {}'.format(role_name, account_number)) return # if we've gotten to this point, load the rest of the role role = Role(get_role_data(dynamo_table, role_id)) old_aa_data_services = [] for aa_service in role.aa_data: if(datetime.datetime.strptime(aa_service['lastUpdated'], '%a, %d %b %Y %H:%M:%S %Z') < datetime.datetime.now() - datetime.timedelta(days=config['repo_requirements']['oldest_aa_data_days'])): old_aa_data_services.append(aa_service['serviceName']) if old_aa_data_services: LOGGER.error('AAData older than threshold for these services: {} (role: {}, account {})'.format( old_aa_data_services, role_name, account_number)) return permissions = roledata._get_role_permissions(role) repoable_permissions = roledata._get_repoable_permissions(account_number, role.role_name, permissions, role.aa_data, role.no_repo_permissions, config['filter_config']['AgeFilter']['minimum_age'], hooks) # if this is a scheduled repo we need to filter out permissions that weren't previously scheduled if scheduled: repoable_permissions = roledata._filter_scheduled_repoable_perms(repoable_permissions, role.scheduled_perms) repoed_policies, deleted_policy_names = roledata._get_repoed_policy(role.policies[-1]['Policy'], repoable_permissions) policies_length = len(json.dumps(repoed_policies)) if policies_length > MAX_AWS_POLICY_SIZE: error = ("Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(role_name, account_number)) LOGGER.error(error) errors.append(error) return if not commit: for name in deleted_policy_names: LOGGER.info('Would delete policy from {} with name {} in account {}'.format( role_name, name, account_number)) if repoed_policies: LOGGER.info('Would replace policies for role {} with: \n{} in account {}'.format( role_name, json.dumps(repoed_policies, indent=2, sort_keys=True), account_number)) return conn = config['connection_iam'] conn['account_number'] = account_number for name in deleted_policy_names: LOGGER.info('Deleting policy with name {} from {} in account {}'.format(name, role.role_name, account_number)) try: delete_role_policy(RoleName=role.role_name, PolicyName=name, **conn) except botocore.exceptions.ClientError as e: error = 'Error deleting policy: {} from role: {} in account {}. Exception: {}'.format( name, role.role_name, account_number, e) LOGGER.error(error) errors.append(error) if repoed_policies: LOGGER.info('Replacing Policies With: \n{} (role: {} account: {})'.format( json.dumps(repoed_policies, indent=2, sort_keys=True), role.role_name, account_number)) for policy_name, policy in repoed_policies.items(): try: put_role_policy(RoleName=role.role_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True), **conn) except botocore.exceptions.ClientError as e: error = 'Exception calling PutRolePolicy on {role}/{policy} in account {account}\n{e}\n'.format( role=role.role_name, policy=policy_name, account=account_number, e=str(e)) LOGGER.error(error) errors.append(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.add_new_policy_version(dynamo_table, role, current_policies, 'Repo') # regardless of whether we're successful we want to unschedule the repo set_role_data(dynamo_table, role.role_id, {'RepoScheduled': 0, 'ScheduledPerms': []}) repokid.hooks.call_hooks(hooks, 'AFTER_REPO', {'role': role}) if not errors: # repos will stay scheduled until they are successful set_role_data(dynamo_table, role.role_id, {'Repoed': datetime.datetime.utcnow().isoformat()}) _update_repoed_description(role.role_name, **conn) _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source='Repo', add_no_repo=False) LOGGER.info('Successfully repoed role: {} in account {}'.format(role.role_name, account_number)) return errors
def _deal_with_policies(role, account_number, config, hooks, scheduled, role_name, dynamo_table, commit, continuing): errors = [] total_permissions, eligible_permissions = roledata._get_role_permissions( role) repoable_permissions = roledata._get_repoable_permissions( account_number, role.role_name, eligible_permissions, role.aa_data, role.no_repo_permissions, config["filter_config"]["AgeFilter"]["minimum_age"], hooks, ) # if this is a scheduled repo we need to filter out permissions that weren't previously scheduled if scheduled: repoable_permissions = roledata._filter_scheduled_repoable_perms( repoable_permissions, role.scheduled_perms) repoed_policies, deleted_policy_names = roledata._get_repoed_policy( role.policies[-1]["Policy"], repoable_permissions) if inline_policies_size_exceeds_maximum(repoed_policies): error = ( "Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(role_name, account_number)) LOGGER.error(error) errors.append(error) continuing = False # if we aren't repoing for some reason, unschedule the role if not continuing: set_role_data(dynamo_table, role.role_id, { "RepoScheduled": 0, "ScheduledPerms": [] }) return if not commit: log_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role_name, account_number) return conn = config["connection_iam"] conn["account_number"] = account_number for name in deleted_policy_names: error = delete_policy(name, role, account_number, conn) if error: LOGGER.error(error) errors.append(error) if repoed_policies: error = replace_policies(repoed_policies, role, account_number, conn) if error: LOGGER.error(error) errors.append(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") # regardless of whether we're successful we want to unschedule the repo set_role_data(dynamo_table, role.role_id, { "RepoScheduled": 0, "ScheduledPerms": [] }) repokid.hooks.call_hooks(hooks, "AFTER_REPO", { "role": role, "errors": errors }) if not errors: # repos will stay scheduled until they are successful set_role_data( dynamo_table, role.role_id, {"Repoed": datetime.datetime.utcnow().isoformat()}, ) update_repoed_description(role.role_name, **conn) partial_update_role_data( role, dynamo_table, account_number, config, conn, hooks, source="Repo", add_no_repo=False, ) LOGGER.info("Successfully repoed role: {} in account {}".format( role.role_name, account_number)) return errors
def rollback_role(account_number, role_name, dynamo_table, config, hooks, selection=None, commit=None): """ Display the historical policy versions for a roll as a numbered list. Restore to a specific version if selected. Indicate changes that will be made and then actually make them if commit is selected. Args: account_number (string) role_name (string) selection (int): which policy version in the list to rollback to commit (bool): actually make the change Returns: errors (list): if any """ errors = [] role_id = find_role_in_cache(dynamo_table, account_number, role_name) if not role_id: message = 'Could not find role with name {} in account {}'.format(role_name, account_number) errors.append(message) LOGGER.warning(message) return errors else: role = Role(get_role_data(dynamo_table, role_id)) # no option selected, display a table of options if not selection: headers = ['Number', 'Source', 'Discovered', 'Permissions', 'Services'] rows = [] for index, policies_version in enumerate(role.policies): policy_permissions = roledata._get_permissions_in_policy(policies_version['Policy']) rows.append([index, policies_version['Source'], policies_version['Discovered'], len(policy_permissions), roledata._get_services_in_permissions(policy_permissions)]) print tabulate(rows, headers=headers) return from cloudaux import CloudAux conn = config['connection_iam'] conn['account_number'] = account_number ca = CloudAux(**conn) current_policies = get_role_inline_policies(role.as_dict(), **conn) if selection: pp = pprint.PrettyPrinter() print "Will restore the following policies:" pp.pprint(role.policies[int(selection)]['Policy']) print "Current policies:" pp.pprint(current_policies) current_permissions = roledata._get_permissions_in_policy(role.policies[-1]['Policy']) selected_permissions = roledata._get_permissions_in_policy(role.policies[int(selection)]['Policy']) restored_permissions = selected_permissions - current_permissions print "\nResore will return these permissions:" print '\n'.join([perm for perm in sorted(restored_permissions)]) if not commit: return False # if we're restoring from a version with fewer policies than we have now, we need to remove them to # complete the restore. To do so we'll store all the policy names we currently have and remove them # from the list as we update. Any policy names left need to be manually removed policies_to_remove = current_policies.keys() for policy_name, policy in role.policies[int(selection)]['Policy'].items(): try: LOGGER.info("Pushing cached policy: {} (role: {} account {})".format( policy_name, role.role_name, account_number)) ca.call('iam.client.put_role_policy', RoleName=role.role_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True)) except botocore.exceptions.ClientError as e: message = "Unable to push policy {}. Error: {} (role: {} account {})".format( policy_name, e.message, role.role_name, account_number) LOGGER.error(message) errors.append(message) else: # remove the policy name if it's in the list try: policies_to_remove.remove(policy_name) except Exception: pass if policies_to_remove: for policy_name in policies_to_remove: try: ca.call('iam.client.delete_role_policy', RoleName=role.role_name, PolicyName=policy_name) except botocore.excpetions.ClientError as e: message = "Unable to delete policy {}. Error: {} (role: {} account {})".format( policy_name, e.message, role.role_name, account_number) LOGGER.error(message) errors.append(message) _update_role_data(role, dynamo_table, account_number, config, conn, hooks, source='Restore', add_no_repo=False) if not errors: LOGGER.info('Successfully restored selected version of role policies (role: {} account: {})'.format( role.role_name, account_number)) return errors
def update_role_cache(account_number, dynamo_table, config, hooks): """ Update data about all roles in a given account: 1) list all the roles and initiate a role object with basic data including name and roleID 2) get inline policies for each of the roles 3) build a list of active roles - we'll want to keep data about roles that may have been deleted in case we need to restore them, so if we used to have a role and now we don't see it we'll mark it inactive 4) update data about the roles in Dynamo 5) mark inactive roles in Dynamo 6) load and instantiate filter plugins 7) for each filter determine the list of roles that it filters 8) update data in Dynamo about filters 9) get Aardvark data for each role 10) update Dynamo with Aardvark data 11) calculate repoable permissions/policies for all the roles 12) update Dynamo with information about how many total and repoable permissions and which services are repoable 13) update stats in Dynamo with basic information like total permissions and which filters are applicable Args: account_number (string): The current account number Repokid is being run against Returns: None """ conn = config['connection_iam'] conn['account_number'] = account_number roles = Roles([Role(role_data) for role_data in list_roles(**conn)]) active_roles = [] LOGGER.info('Updating role data for account {}'.format(account_number)) for role in tqdm(roles): role.account = account_number current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} active_roles.append(role.role_id) roledata.update_role_data(dynamo_table, account_number, role, current_policies) LOGGER.info('Finding inactive roles in account {}'.format(account_number)) roledata.find_and_mark_inactive(dynamo_table, account_number, active_roles) LOGGER.info('Filtering roles') plugins = FilterPlugins() # Blacklist needs to know the current account config['filter_config']['BlacklistFilter']['current_account'] = account_number for plugin_path in config.get('active_filters'): plugin_name = plugin_path.split(':')[1] plugins.load_plugin(plugin_path, config=config['filter_config'].get(plugin_name, None)) for plugin in plugins.filter_plugins: filtered_list = plugin.apply(roles) class_name = plugin.__class__.__name__ for filtered_role in filtered_list: LOGGER.info('Role {} filtered by {}'.format(filtered_role.role_name, class_name)) filtered_role.disqualified_by.append(class_name) for role in roles: set_role_data(dynamo_table, role.role_id, {'DisqualifiedBy': role.disqualified_by}) LOGGER.info('Getting data from Aardvark for account {}'.format(account_number)) aardvark_data = _get_aardvark_data(config['aardvark_api_location'], account_number=account_number) LOGGER.info('Updating roles with Aardvark data in account {}'.format(account_number)) for role in roles: try: role.aa_data = aardvark_data[role.arn] except KeyError: LOGGER.warning('Aardvark data not found for role: {} ({})'.format(role.role_id, role.role_name)) else: set_role_data(dynamo_table, role.role_id, {'AAData': role.aa_data}) LOGGER.info('Calculating repoable permissions and services for account {}'.format(account_number)) roledata._calculate_repo_scores(roles, config['filter_config']['AgeFilter']['minimum_age'], hooks) for role in roles: LOGGER.debug('Role {} in account {} has\nrepoable permissions: {}\nrepoable services:'.format( role.role_name, account_number, role.repoable_permissions, role.repoable_services )) set_role_data(dynamo_table, role.role_id, {'TotalPermissions': role.total_permissions, 'RepoablePermissions': role.repoable_permissions, 'RepoableServices': role.repoable_services}) LOGGER.info('Updating stats in account {}'.format(account_number)) roledata.update_stats(dynamo_table, roles, source='Scan')
def remove_permissions_from_role( account_number, permissions, role, role_id, dynamo_table, config, hooks, commit=False, ): """Remove the list of permissions from the provided role. Args: account_number (string) permissions (list<string>) role (Role object) role_id (string) commit (bool) Returns: None """ repoed_policies, deleted_policy_names = roledata._get_repoed_policy( role.policies[-1]["Policy"], permissions) if inline_policies_size_exceeds_maximum(repoed_policies): LOGGER.error( "Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(role.role_name, account_number)) return if not commit: log_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role.role_name, account_number) return conn = config["connection_iam"] conn["account_number"] = account_number for name in deleted_policy_names: error = delete_policy(name, role, account_number, conn) if error: LOGGER.error(error) if repoed_policies: error = replace_policies(repoed_policies, role, account_number, conn) if error: LOGGER.error(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") set_role_data(dynamo_table, role.role_id, {"Repoed": datetime.datetime.utcnow().isoformat()}) update_repoed_description(role.role_name, **conn) partial_update_role_data( role, dynamo_table, account_number, config, conn, hooks, source="ManualPermissionRepo", add_no_repo=False, ) LOGGER.info( "Successfully removed {permissions} from role: {role} in account {account_number}" .format(permissions=permissions, role=role.role_name, account_number=account_number))
def rollback_role(account_number, role_name, selection=None, commit=None): role_data = _find_role_in_cache(role_name) if not role_data: LOGGER.error("Couldn't find role ({}) in cache".format(role_name)) return # no option selected, display a table of options if not selection: headers = [ 'Number', 'Source', 'Discovered', 'Policy Length', 'Policy Contents' ] rows = [] for index, policies_version in enumerate(role_data['Policies']): rows.append([ index, policies_version['Source'], policies_version['Discovered'], len(str(policies_version['Policy'])), str(policies_version['Policy'])[:50] ]) print tabulate(rows, headers=headers) return from cloudaux import CloudAux conn = CONFIG['connection_iam'] conn['account_number'] = account_number ca = CloudAux(**conn) current_policies = get_role_inline_policies(role_data, **conn) if selection and not commit: pp = pprint.PrettyPrinter() print "Will restore the following policies:" pp.pprint(role_data['Policies'][int(selection)]['Policy']) print "Current policies:" pp.pprint(current_policies) return # if we're restoring from a version with fewer policies than we have now, we need to remove them to # complete the restore. To do so we'll store all the policy names we currently have and remove them # from the list as we update. Any policy names left need to be manually removed policies_to_remove = current_policies.keys() for policy_name, policy in role_data['Policies'][int( selection)]['Policy'].items(): try: LOGGER.info("Pushing cached policy: {}".format(policy_name)) ca.call('iam.client.put_role_policy', RoleName=role_data['RoleName'], PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True)) except Exception as e: LOGGER.error("Unable to push policy {}. Error: {}".format( policy_name, e.message)) else: # remove the policy name if it's in the list try: policies_to_remove.remove(policy_name) except: pass if policies_to_remove: for policy_name in policies_to_remove: try: ca.call('iam.client.delete_role_policy', RoleName=role_data['RoleName'], PolicyName=policy_name) except Exception as e: LOGGER.error("Unable to delete policy {}. Error: {}".format( policy_name, e.message)) role_data['policies'] = get_role_inline_policies(role_data, **conn) or {} roledata.add_new_policy_version(role_data, 'Restore') LOGGER.info('Successfully restored selected version of role policies')
def repo_role(account_number, role_name, commit=False): role_data = _find_role_in_cache(role_name) if not role_data: LOGGER.error('Could not find role with name {}'.format(role_name)) return disqualified_by = role_data.get('DisqualifiedBy', []) if len(disqualified_by) > 0: LOGGER.info( 'Cannot repo role {} because it is being disqualified by: {}'. format(role_name, disqualified_by)) return if 'AAData' not in role_data: LOGGER.warn('ARN not found in Access Advisor: {}'.format( role_data['Arn'])) return old_aa_data_services = [] for aa_service in role_data['AAData']: if (datetime.datetime.strptime(aa_service['lastUpdated'], '%a, %d %B %Y %H:%M:%S %Z') < datetime.datetime.now() - datetime.timedelta( days=CONFIG['repo_requirements']['oldest_aa_data_days'])): old_aa_data_services.append(aa_service['serviceName']) if old_aa_data_services: LOGGER.error( 'AAData older than threshold for these services: {}'.format( old_aa_data_services)) return permissions = _get_role_permissions(role_data) repoable_permissions = _get_repoable_permissions(permissions, role_data) repoed_policies, deleted_policy_names = _get_repoed_policy( role_data, repoable_permissions) policies_length = len(json.dumps(repoed_policies)) if not commit: for name in deleted_policy_names: LOGGER.info('Would delete policy from {} with name {}'.format( role_name, name)) if repoed_policies: LOGGER.info('Would replace policies for role {} with: \n{}'.format( role_name, json.dumps(repoed_policies, indent=2, sort_keys=True))) if policies_length > MAX_AWS_POLICY_SIZE: LOGGER.error( "Policies would exceed the AWS size limit after repo for role: {}. " "Please manually minify.".format(role_name)) return from cloudaux import CloudAux conn = CONFIG['connection_iam'] conn['account_number'] = account_number ca = CloudAux(**conn) if policies_length > MAX_AWS_POLICY_SIZE: LOGGER.error( "Policies would exceed the AWS size limit after repo for role: {}. " "Please manually minify.".format(role_name)) return for name in deleted_policy_names: LOGGER.info('Deleting policy with name {} from {}'.format( name, role_data['RoleName'])) ca.call('iam.client.delete_role_policy', RoleName=role_data['RoleName'], PolicyName=name) if repoed_policies: LOGGER.info('Replacing Policies With: \n{}'.format( json.dumps(repoed_policies, indent=2, sort_keys=True))) for policy_name, policy in repoed_policies.items(): try: ca.call('iam.client.put_role_policy', RoleName=role_data['RoleName'], PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True)) except Exception as e: LOGGER.error( 'Exception calling PutRolePolicy on {role}/{policy}\n{e}\n' .format(role=role_data['RoleName'], policy=policy_name, e=str(e))) return role_data['policies'] = get_role_inline_policies(role_data, **conn) or {} roledata.add_new_policy_version(role_data, 'Repo') roledata.set_repoed(role_data['RoleId']) # update total permissions count for stats permissions_count = len(_get_role_permissions(role_data)) roledata.update_total_permissions(role_data['RoleId'], permissions_count) roledata.update_stats(source='Repo', roleID=role_data['RoleId']) LOGGER.info('Successfully repoed role: {}'.format(role_data['RoleName']))
def get_inline_policies(role, **conn): return get_role_inline_policies(role, **conn)
def _repo_role( account_number, role_name, dynamo_table, config, hooks, commit=False, scheduled=False, ): """ Calculate what repoing can be done for a role and then actually do it if commit is set 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data 2) Get the role's current permissions, repoable permissions, and the new policy if it will change 3) Make the changes if commit is set Args: account_number (string) role_name (string) commit (bool) Returns: None """ errors = [] role_id = find_role_in_cache(dynamo_table, account_number, role_name) # only load partial data that we need to determine if we should keep going role_data = get_role_data( dynamo_table, role_id, fields=["DisqualifiedBy", "AAData", "RepoablePermissions", "RoleName"], ) if not role_data: LOGGER.warn("Could not find role with name {}".format(role_name)) return else: role = Role(role_data) continuing = True if len(role.disqualified_by) > 0: LOGGER.info( "Cannot repo role {} in account {} because it is being disqualified by: {}" .format(role_name, account_number, role.disqualified_by)) continuing = False if not role.aa_data: LOGGER.warning("ARN not found in Access Advisor: {}".format(role.arn)) continuing = False if not role.repoable_permissions: LOGGER.info("No permissions to repo for role {} in account {}".format( role_name, account_number)) continuing = False # if we've gotten to this point, load the rest of the role role = Role(get_role_data(dynamo_table, role_id)) old_aa_data_services = [] for aa_service in role.aa_data: if datetime.datetime.strptime( aa_service["lastUpdated"], "%a, %d %b %Y %H:%M:%S %Z" ) < datetime.datetime.now() - datetime.timedelta( days=config["repo_requirements"]["oldest_aa_data_days"]): old_aa_data_services.append(aa_service["serviceName"]) if old_aa_data_services: LOGGER.error( "AAData older than threshold for these services: {} (role: {}, account {})" .format(old_aa_data_services, role_name, account_number), exc_info=True, ) continuing = False total_permissions, eligible_permissions = roledata._get_role_permissions( role) repoable_permissions = roledata._get_repoable_permissions( account_number, role.role_name, eligible_permissions, role.aa_data, role.no_repo_permissions, config["filter_config"]["AgeFilter"]["minimum_age"], hooks, ) # if this is a scheduled repo we need to filter out permissions that weren't previously scheduled if scheduled: repoable_permissions = roledata._filter_scheduled_repoable_perms( repoable_permissions, role.scheduled_perms) repoed_policies, deleted_policy_names = roledata._get_repoed_policy( role.policies[-1]["Policy"], repoable_permissions) if inline_policies_size_exceeds_maximum(repoed_policies): error = ( "Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(role_name, account_number)) LOGGER.error(error) errors.append(error) continuing = False # if we aren't repoing for some reason, unschedule the role if not continuing: set_role_data(dynamo_table, role.role_id, { "RepoScheduled": 0, "ScheduledPerms": [] }) return if not commit: log_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role_name, account_number) return conn = config["connection_iam"] conn["account_number"] = account_number for name in deleted_policy_names: error = delete_policy(name, role, account_number, conn) if error: LOGGER.error(error) errors.append(error) if repoed_policies: error = replace_policies(repoed_policies, role, account_number, conn) if error: LOGGER.error(error) errors.append(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.add_new_policy_version(dynamo_table, role, current_policies, "Repo") # regardless of whether we're successful we want to unschedule the repo set_role_data(dynamo_table, role.role_id, { "RepoScheduled": 0, "ScheduledPerms": [] }) repokid.hooks.call_hooks(hooks, "AFTER_REPO", { "role": role, "errors": errors }) if not errors: # repos will stay scheduled until they are successful set_role_data( dynamo_table, role.role_id, {"Repoed": datetime.datetime.utcnow().isoformat()}, ) update_repoed_description(role.role_name, **conn) partial_update_role_data( role, dynamo_table, account_number, config, conn, hooks, source="Repo", add_no_repo=False, ) LOGGER.info("Successfully repoed role: {} in account {}".format( role.role_name, account_number)) return errors
def rollback_role(account_number, role_name, selection=None, commit=None): """ Display the historical policy versions for a roll as a numbered list. Restore to a specific version if selected. Indicate changes that will be made and then actually make them if commit is selected. Args: account_number (string) role_name (string) selection (int): which policy version in the list to rollback to commit (bool): actually make the change Returns: None """ role_data = _find_role_in_cache(account_number, role_name) if not role_data: LOGGER.warn('Could not find role with name {}'.format(role_name)) return else: role = Role(role_data) # no option selected, display a table of options if not selection: headers = [ 'Number', 'Source', 'Discovered', 'Policy Length', 'Policy Contents' ] rows = [] for index, policies_version in enumerate(role.policies): rows.append([ index, policies_version['Source'], policies_version['Discovered'], len(str(policies_version['Policy'])), str(policies_version['Policy'])[:50] ]) print tabulate(rows, headers=headers) return from cloudaux import CloudAux conn = CONFIG['connection_iam'] conn['account_number'] = account_number ca = CloudAux(**conn) current_policies = get_role_inline_policies(role.as_dict(), **conn) if selection and not commit: pp = pprint.PrettyPrinter() print "Will restore the following policies:" pp.pprint(role.policies[int(selection)]['Policy']) print "Current policies:" pp.pprint(current_policies) return # if we're restoring from a version with fewer policies than we have now, we need to remove them to # complete the restore. To do so we'll store all the policy names we currently have and remove them # from the list as we update. Any policy names left need to be manually removed policies_to_remove = current_policies.keys() for policy_name, policy in role.policies[int(selection)]['Policy'].items(): try: LOGGER.info("Pushing cached policy: {}".format(policy_name)) ca.call('iam.client.put_role_policy', RoleName=role.role_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True)) except botocore.excpetion.ClientError as e: LOGGER.error("Unable to push policy {}. Error: {}".format( policy_name, e.message)) else: # remove the policy name if it's in the list try: policies_to_remove.remove(policy_name) except: pass if policies_to_remove: for policy_name in policies_to_remove: try: ca.call('iam.client.delete_role_policy', RoleName=role.role_name, PolicyName=policy_name) except botocore.excpetion.ClientError as e: LOGGER.error("Unable to delete policy {}. Error: {}".format( policy_name, e.message)) # TODO: possibly update the total and repoable permissions here, we'd have to get Aardvark and all that current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.add_new_policy_version(role, current_policies, 'Restore') role.total_permissions = len(roledata._get_role_permissions(role)) # update stats roledata.update_stats([role], source='Restore') LOGGER.info('Successfully restored selected version of role policies')
def repo_role(account_number, role_name, commit=False): """ Calculate what repoing can be done for a role and then actually do it if commit is set 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data 2) Get the role's current permissions, repoable permissions, and the new policy if it will change 3) Make the changes if commit is set Args: account_number (string) role_name (string) commit (bool) Returns: None """ errors = [] role_data = _find_role_in_cache(account_number, role_name) if not role_data: LOGGER.warn('Could not find role with name {}'.format(role_name)) return else: role = Role(role_data) if len(role.disqualified_by) > 0: LOGGER.info( 'Cannot repo role {} because it is being disqualified by: {}'. format(role_name, role.disqualified_by)) return if not role.aa_data: LOGGER.warn('ARN not found in Access Advisor: {}'.format(role.arn)) return if not role.repoable_permissions: LOGGER.info('No permissions to repo for role {}'.format(role_name)) return old_aa_data_services = [] for aa_service in role.aa_data: if (datetime.datetime.strptime(aa_service['lastUpdated'], '%a, %d %b %Y %H:%M:%S %Z') < datetime.datetime.now() - datetime.timedelta( days=CONFIG['repo_requirements']['oldest_aa_data_days'])): old_aa_data_services.append(aa_service['serviceName']) if old_aa_data_services: LOGGER.error( 'AAData older than threshold for these services: {}'.format( old_aa_data_services)) return permissions = roledata._get_role_permissions(role) repoable_permissions = roledata._get_repoable_permissions( permissions, role.aa_data, role.no_repo_permissions) repoed_policies, deleted_policy_names = roledata._get_repoed_policy( role.policies[-1]['Policy'], repoable_permissions) policies_length = len(json.dumps(repoed_policies)) if not commit: for name in deleted_policy_names: LOGGER.info('Would delete policy from {} with name {}'.format( role_name, name)) if repoed_policies: LOGGER.info('Would replace policies for role {} with: \n{}'.format( role_name, json.dumps(repoed_policies, indent=2, sort_keys=True))) if policies_length > MAX_AWS_POLICY_SIZE: LOGGER.error( "Policies would exceed the AWS size limit after repo for role: {}. " "Please manually minify.".format(role_name)) return from cloudaux import CloudAux conn = CONFIG['connection_iam'] conn['account_number'] = account_number ca = CloudAux(**conn) if policies_length > MAX_AWS_POLICY_SIZE: LOGGER.error( "Policies would exceed the AWS size limit after repo for role: {}. " "Please manually minify.".format(role_name)) return for name in deleted_policy_names: LOGGER.info('Deleting policy with name {} from {}'.format( name, role.role_name)) try: ca.call('iam.client.delete_role_policy', RoleName=role.role_name, PolicyName=name) except botocore.exceptions.ClientError as e: error = 'Error deleting policy: {} from role: {}. Exception: {}'.format( name, role.role_name, e) LOGGER.error(error) errors.append(error) if repoed_policies: LOGGER.info('Replacing Policies With: \n{}'.format( json.dumps(repoed_policies, indent=2, sort_keys=True))) for policy_name, policy in repoed_policies.items(): try: ca.call('iam.client.put_role_policy', RoleName=role.role_name, PolicyName=policy_name, PolicyDocument=json.dumps(policy, indent=2, sort_keys=True)) except botocore.exceptions.ClientError as e: error = 'Exception calling PutRolePolicy on {role}/{policy}\n{e}\n'.format( role=role.role_name, policy=policy_name, e=str(e)) LOGGER.error(error) errors.append(error) current_policies = get_role_inline_policies(role.as_dict(), **conn) or {} roledata.add_new_policy_version(role, current_policies, 'Repo') if not errors: roledata.set_repoed(role.role_id) # update total and repoable permissions and services role.total_permissions = len(roledata._get_role_permissions(role)) role.repoable_permissions = 0 role.repoable_services = [] roledata.update_repoable_data([role]) # update stats roledata.update_stats([role], source='Repo') LOGGER.info('Successfully repoed role: {}'.format(role.role_name)) return errors
def update_role_cache(account_number): """ Update data about all roles in a given account: 1) list all the roles and initiate a role object with basic data including name and roleID 2) get inline policies for each of the roles 3) build a list of active roles - we'll want to keep data about roles that may have been deleted in case we need to restore them, so if we used to have a role and now we don't see it we'll mark it inactive 4) update data about the roles in Dynamo 5) mark inactive roles in Dynamo 6) load and instantiate filter plugins 7) for each filter determine the list of roles that it filters 8) update data in Dynamo about filters 9) get Aardvark data for each role 10) update Dynamo with Aardvark data 11) calculate repoable permissions/policies for all the roles 12) update Dynamo with information about how many total and repoable permissions and which services are repoable 13) update stats in Dynamo with basic information like total permissions and which filters are applicable Args: account_number (string): The current account number Repokid is being run against Returns: None """ conn = CONFIG['connection_iam'] conn['account_number'] = account_number roles = Roles([Role(role_data) for role_data in list_roles(**conn)]) active_roles = [] LOGGER.info('Updating role data for account {}'.format(account_number)) for role in tqdm(roles): current_policies = get_role_inline_policies(role.as_dict(), ** conn) or {} active_roles.append(role.role_id) roledata.update_role_data(role, current_policies) LOGGER.info('Finding inactive accounts') roledata.find_and_mark_inactive(account_number, active_roles) LOGGER.info('Filtering roles') plugins = FilterPlugins() for plugin in CONFIG.get('active_filters'): plugins.load_plugin(plugin) for plugin in plugins.filter_plugins: filtered_list = plugin.apply(roles) class_name = plugin.__class__.__name__ for role in filtered_list: LOGGER.info('Role {} filtered by {}'.format( role.role_name, class_name)) roles.get_by_id(role.role_id).disqualified_by.append(class_name) roledata.update_filtered_roles(roles) LOGGER.info('Getting data from Aardvark') aardvark_data = _get_aardvark_data(account_number) LOGGER.info('Updating with Aardvark data') roledata.update_aardvark_data(aardvark_data, roles) LOGGER.info('Calculating repoable permissions and services') roledata._calculate_repo_scores(roles) roledata.update_repoable_data(roles) LOGGER.info('Updating stats') roledata.update_stats(roles, source='Scan')
def repo(self, hooks: RepokidHooks, commit: bool = False, scheduled: bool = False) -> List[str]: errors: List[str] = [] eligible, reason = self.is_eligible_for_repo() if not eligible: errors.append( f"Role {self.role_name} not eligible for repo: {reason}") return errors self.calculate_repo_scores( self.config["filter_config"]["AgeFilter"]["minimum_age"], hooks # type: ignore ) try: repoed_policies, deleted_policy_names = self.get_repoed_policy( scheduled=scheduled) except MissingRepoableServices as e: errors.append(f"Role {self.role_name} cannot be repoed: {e}") return errors if inline_policies_size_exceeds_maximum(repoed_policies): error = ( "Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(self.role_name, self.account)) logger.error(error) errors.append(error) self.repo_scheduled = 0 self.scheduled_perms = [] self.store(["repo_scheduled", "scheduled_perms"]) return errors if not commit: log_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, self.role_name, self.account) return errors conn = self.config["connection_iam"] # type: ignore conn["account_number"] = self.account for name in deleted_policy_names: try: delete_policy(name, self.role_name, self.account, conn) except IAMError as e: logger.error(e) errors.append(str(e)) if repoed_policies: try: replace_policies(repoed_policies, self.role_name, self.account, conn) except IAMError as e: logger.error(e) errors.append(str(e)) current_policies = (get_role_inline_policies(self.dict(by_alias=True), **conn) or {}) self.add_policy_version(current_policies, source="Repo") # regardless of whether we're successful we want to unschedule the repo self.repo_scheduled = 0 self.scheduled_perms = [] call_hooks(hooks, "AFTER_REPO", {"role": self, "errors": errors}) if not errors: # repos will stay scheduled until they are successful self.repoed = datetime.datetime.now( tz=datetime.timezone.utc).isoformat() update_repoed_description(self.role_name, conn) logger.info("Successfully repoed role: {} in account {}".format( self.role_name, self.account)) try: self.store() except RoleStoreError: logger.exception("failed to store role after repo", exc_info=True) return errors
def remove_permissions(self, permissions: List[str], hooks: RepokidHooks, commit: bool = False) -> None: """Remove the list of permissions from the provided role. Args: account_number (string) permissions (list<string>) role (Role object) role_id (string) commit (bool) Returns: None """ ( repoed_policies, deleted_policy_names, ) = get_repoed_policy(self.policies[-1]["Policy"], set(permissions)) if inline_policies_size_exceeds_maximum(repoed_policies): logger.error( "Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(self.role_name, self.account)) return if not commit: log_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, self.role_name, self.account) return conn = self.config["connection_iam"] # type: ignore conn["account_number"] = self.account for name in deleted_policy_names: try: delete_policy(name, self.role_name, self.account, conn) except IAMError as e: logger.error(e) if repoed_policies: try: replace_policies(repoed_policies, self.role_name, self.account, conn) except IAMError as e: logger.error(e) current_policies = get_role_inline_policies(self.dict(), **conn) or {} self.add_policy_version(current_policies, "Repo") self.repoed = datetime.datetime.now( tz=datetime.timezone.utc).isoformat() update_repoed_description(self.role_name, conn) self.gather_role_data( hooks, current_policies=current_policies, source="ManualPermissionRepo", add_no_repo=False, ) logger.info( "Successfully removed {permissions} from role: {role} in account {account_number}" .format( permissions=permissions, role=self.role_name, account_number=self.account, ))
def partial_update_role_data(role, dynamo_table, account_number, config, conn, hooks, source, add_no_repo=True): """ Perform a scaled down version of role update, this is used to get an accurate count of repoable permissions after a rollback or repo. Does update: - Policies - Aardvark data - Total permissions - Repoable permissions - Repoable services - Stats Does not update: - Filters - Active/inactive roles Args: role (Role) dynamo_table account_number config conn (dict) hooks source: repo, rollback, etc add_no_repo: if set to True newly discovered permissions will be added to no repo list Returns: None """ current_policies = get_role_inline_policies(role.dict(), **conn) or {} update_role_data( dynamo_table, account_number, role, current_policies, source=source, add_no_repo=add_no_repo, ) aardvark_data = get_aardvark_data(config["aardvark_api_location"], arn=role.arn) if not aardvark_data: return batch_processing = config.get("query_role_data_in_batch", False) batch_size = config.get("batch_processing_size", 100) role.aa_data = aardvark_data[role.arn] _calculate_repo_scores( [role], config["filter_config"]["AgeFilter"]["minimum_age"], hooks, batch_processing, batch_size, ) set_role_data( dynamo_table, role.role_id, { "AAData": role.aa_data, "TotalPermissions": role.total_permissions, "RepoablePermissions": role.repoable_permissions, "RepoableServices": role.repoable_services, }, ) update_stats(dynamo_table, [role], source=source) # TODO update
def _repo_role( account_number: str, role_name: str, config: RepokidConfig, hooks: RepokidHooks, commit: bool = False, scheduled: bool = False, ) -> List[str]: """ Calculate what repoing can be done for a role and then actually do it if commit is set 1) Check that a role exists, it isn't being disqualified by a filter, and that is has fresh AA data 2) Get the role's current permissions, repoable permissions, and the new policy if it will change 3) Make the changes if commit is set Args: account_number (string) role_name (string) commit (bool) Returns: None """ errors: List[str] = [] role_id = find_role_in_cache(role_name, account_number) # only load partial data that we need to determine if we should keep going role = Role(role_id=role_id) role.fetch() continuing = True eligible, reason = role.is_eligible_for_repo() if not eligible: errors.append(f"Role {role_name} not eligible for repo: {reason}") return errors role.calculate_repo_scores( config["filter_config"]["AgeFilter"]["minimum_age"], hooks) repoed_policies, deleted_policy_names = role.get_repoed_policy( scheduled=scheduled) if inline_policies_size_exceeds_maximum(repoed_policies): error = ( "Policies would exceed the AWS size limit after repo for role: {} in account {}. " "Please manually minify.".format(role_name, account_number)) LOGGER.error(error) errors.append(error) continuing = False # if we aren't repoing for some reason, unschedule the role if not continuing: role.repo_scheduled = 0 role.scheduled_perms = [] role.store(["repo_scheduled", "scheduled_perms"]) return errors if not commit: log_deleted_and_repoed_policies(deleted_policy_names, repoed_policies, role_name, account_number) return errors conn = config["connection_iam"] conn["account_number"] = account_number for name in deleted_policy_names: try: delete_policy(name, role, account_number, conn) except IAMError as e: LOGGER.error(e) errors.append(str(e)) if repoed_policies: try: replace_policies(repoed_policies, role, account_number, conn) except IAMError as e: LOGGER.error(e) errors.append(str(e)) current_policies = get_role_inline_policies(role.dict(by_alias=True), ** conn) or {} role.add_policy_version(current_policies, source="Repo") # regardless of whether we're successful we want to unschedule the repo role.repo_scheduled = 0 role.scheduled_perms = [] repokid.hooks.call_hooks(hooks, "AFTER_REPO", { "role": role, "errors": errors }) if not errors: # repos will stay scheduled until they are successful role.repoed = datetime.datetime.now( tz=datetime.timezone.utc).isoformat() update_repoed_description(role.role_name, conn) role.gather_role_data(current_policies, hooks, source="Repo", add_no_repo=False) LOGGER.info("Successfully repoed role: {} in account {}".format( role.role_name, account_number)) role.store() return errors