def _update_role_cache(
account_number: str,
config: RepokidConfig,
hooks: RepokidHooks,
) -> None:
"""
Update data about all roles in a given account:
1) list all the roles and initiate a role object with basic data including name and roleID
2) get inline policies for each of the roles
3) build a list of active roles - we'll want to keep data about roles that may have been deleted in case we
need to restore them, so if we used to have a role and now we don't see it we'll mark it inactive
4) update data about the roles in Dynamo
5) mark inactive roles in Dynamo
6) load and instantiate filter plugins
7) for each filter determine the list of roles that it filters
8) update data in Dynamo about filters
9) get Aardvark data for each role
10) update Dynamo with Aardvark data
11) calculate repoable permissions/policies for all the roles
12) update Dynamo with information about how many total and repoable permissions and which services are repoable
13) update stats in Dynamo with basic information like total permissions and which filters are applicable
Args:
account_number (string): The current account number Repokid is being run against
Returns:
None
"""
access_advisor_datasource = AccessAdvisorDatasource()
access_advisor_datasource.seed(account_number)
iam_datasource = IAMDatasource()
role_ids = iam_datasource.seed(account_number)
# We only iterate over the newly-seeded data (`role_ids`) so we don't duplicate work for runs on multiple accounts
roles = RoleList([Role(**iam_datasource[role_id]) for role_id in role_ids])
LOGGER.info("Updating role data for account {}".format(account_number))
for role in tqdm(roles):
role.account = account_number
role.gather_role_data(hooks, config=config, source="Scan", store=False)
LOGGER.info("Finding inactive roles in account {}".format(account_number))
find_and_mark_inactive(account_number, roles)
LOGGER.info("Filtering roles")
plugins = get_filter_plugins(account_number, config=config)
for plugin in plugins.filter_plugins:
filtered_list = plugin.apply(roles)
class_name = plugin.__class__.__name__
for filtered_role in filtered_list:
LOGGER.debug("Role {} filtered by {}".format(
filtered_role.role_name, class_name))
# There may be existing duplicate records, so we do a dance here to dedupe them.
disqualified_by = set(filtered_role.disqualified_by)
disqualified_by.add(class_name)
filtered_role.disqualified_by = list(disqualified_by)
for role in roles:
LOGGER.debug(
"Role {} in account {} has\nrepoable permissions: {}\nrepoable services: {}"
.format(
role.role_name,
account_number,
role.repoable_permissions,
role.repoable_services,
))
LOGGER.info(
"Storing updated role data in account {}".format(account_number))
roles.store()
def update_role_cache(account_number, dynamo_table, config, hooks):
"""
Update data about all roles in a given account:
1) list all the roles and initiate a role object with basic data including name and roleID
2) get inline policies for each of the roles
3) build a list of active roles - we'll want to keep data about roles that may have been deleted in case we
need to restore them, so if we used to have a role and now we don't see it we'll mark it inactive
4) update data about the roles in Dynamo
5) mark inactive roles in Dynamo
6) load and instantiate filter plugins
7) for each filter determine the list of roles that it filters
8) update data in Dynamo about filters
9) get Aardvark data for each role
10) update Dynamo with Aardvark data
11) calculate repoable permissions/policies for all the roles
12) update Dynamo with information about how many total and repoable permissions and which services are repoable
13) update stats in Dynamo with basic information like total permissions and which filters are applicable
Args:
account_number (string): The current account number Repokid is being run against
Returns:
None
"""
conn = config["connection_iam"]
conn["account_number"] = account_number
LOGGER.info(
"Getting current role data for account {} (this may take a while for large accounts)"
.format(account_number))
role_data = get_account_authorization_details(filter="Role", **conn)
role_data_by_id = {item["RoleId"]: item for item in role_data}
# convert policies list to dictionary to maintain consistency with old call which returned a dict
for _, data in role_data_by_id.items():
data["RolePolicyList"] = {
item["PolicyName"]: item["PolicyDocument"]
for item in data["RolePolicyList"]
}
roles = Roles([Role(rd) for rd in role_data])
active_roles = []
LOGGER.info("Updating role data for account {}".format(account_number))
for role in tqdm(roles):
role.account = account_number
current_policies = role_data_by_id[role.role_id]["RolePolicyList"]
active_roles.append(role.role_id)
roledata.update_role_data(dynamo_table, account_number, role,
current_policies)
LOGGER.info("Finding inactive roles in account {}".format(account_number))
roledata.find_and_mark_inactive(dynamo_table, account_number, active_roles)
LOGGER.info("Filtering roles")
plugins = FilterPlugins()
# Blocklist needs to know the current account
filter_config = config["filter_config"]
blocklist_filter_config = filter_config.get(
"BlocklistFilter", filter_config.get("BlacklistFilter"))
blocklist_filter_config["current_account"] = account_number
for plugin_path in config.get("active_filters"):
plugin_name = plugin_path.split(":")[1]
if plugin_name == "ExclusiveFilter":
# ExclusiveFilter plugin active; try loading its config. Also, it requires the current account, so add it.
exclusive_filter_config = filter_config.get("ExclusiveFilter", {})
exclusive_filter_config["current_account"] = account_number
plugins.load_plugin(plugin_path,
config=config["filter_config"].get(
plugin_name, None))
for plugin in plugins.filter_plugins:
filtered_list = plugin.apply(roles)
class_name = plugin.__class__.__name__
for filtered_role in filtered_list:
LOGGER.info("Role {} filtered by {}".format(
filtered_role.role_name, class_name))
filtered_role.disqualified_by.append(class_name)
for role in roles:
set_role_data(dynamo_table, role.role_id,
{"DisqualifiedBy": role.disqualified_by})
LOGGER.info(
"Getting data from Aardvark for account {}".format(account_number))
aardvark_data = get_aardvark_data(config["aardvark_api_location"],
account_number=account_number)
LOGGER.info("Updating roles with Aardvark data in account {}".format(
account_number))
for role in roles:
try:
role.aa_data = aardvark_data[role.arn]
except KeyError:
LOGGER.warning("Aardvark data not found for role: {} ({})".format(
role.role_id, role.role_name))
else:
set_role_data(dynamo_table, role.role_id, {"AAData": role.aa_data})
LOGGER.info(
"Calculating repoable permissions and services for account {}".format(
account_number))
batch_processing = config.get("query_role_data_in_batch", False)
batch_size = config.get("batch_processing_size", 100)
roledata._calculate_repo_scores(
roles,
config["filter_config"]["AgeFilter"]["minimum_age"],
hooks,
batch_processing,
batch_size,
)
for role in roles:
LOGGER.debug(
"Role {} in account {} has\nrepoable permissions: {}\nrepoable services: {}"
.format(
role.role_name,
account_number,
role.repoable_permissions,
role.repoable_services,
))
set_role_data(
dynamo_table,
role.role_id,
{
"TotalPermissions": role.total_permissions,
"RepoablePermissions": role.repoable_permissions,
"RepoableServices": role.repoable_services,
},
)
LOGGER.info("Updating stats in account {}".format(account_number))
roledata.update_stats(dynamo_table, roles, source="Scan")
def update_role_cache(account_number, dynamo_table, config, hooks):
"""
Update data about all roles in a given account:
1) list all the roles and initiate a role object with basic data including name and roleID
2) get inline policies for each of the roles
3) build a list of active roles - we'll want to keep data about roles that may have been deleted in case we
need to restore them, so if we used to have a role and now we don't see it we'll mark it inactive
4) update data about the roles in Dynamo
5) mark inactive roles in Dynamo
6) load and instantiate filter plugins
7) for each filter determine the list of roles that it filters
8) update data in Dynamo about filters
9) get Aardvark data for each role
10) update Dynamo with Aardvark data
11) calculate repoable permissions/policies for all the roles
12) update Dynamo with information about how many total and repoable permissions and which services are repoable
13) update stats in Dynamo with basic information like total permissions and which filters are applicable
Args:
account_number (string): The current account number Repokid is being run against
Returns:
None
"""
conn = config['connection_iam']
conn['account_number'] = account_number
LOGGER.info('Getting current role data for account {} (this may take a while for large accounts)'.format(
account_number))
role_data = get_account_authorization_details(filter='Role', **conn)
role_data_by_id = {item['RoleId']: item for item in role_data}
# convert policies list to dictionary to maintain consistency with old call which returned a dict
for _, data in role_data_by_id.items():
data['RolePolicyList'] = {item['PolicyName']: item['PolicyDocument'] for item in data['RolePolicyList']}
roles = Roles([Role(rd) for rd in role_data])
active_roles = []
LOGGER.info('Updating role data for account {}'.format(account_number))
for role in tqdm(roles):
role.account = account_number
current_policies = role_data_by_id[role.role_id]['RolePolicyList']
active_roles.append(role.role_id)
roledata.update_role_data(dynamo_table, account_number, role, current_policies)
LOGGER.info('Finding inactive roles in account {}'.format(account_number))
roledata.find_and_mark_inactive(dynamo_table, account_number, active_roles)
LOGGER.info('Filtering roles')
plugins = FilterPlugins()
# Blacklist needs to know the current account
config['filter_config']['BlacklistFilter']['current_account'] = account_number
for plugin_path in config.get('active_filters'):
plugin_name = plugin_path.split(':')[1]
plugins.load_plugin(plugin_path, config=config['filter_config'].get(plugin_name, None))
for plugin in plugins.filter_plugins:
filtered_list = plugin.apply(roles)
class_name = plugin.__class__.__name__
for filtered_role in filtered_list:
LOGGER.info('Role {} filtered by {}'.format(filtered_role.role_name, class_name))
filtered_role.disqualified_by.append(class_name)
for role in roles:
set_role_data(dynamo_table, role.role_id, {'DisqualifiedBy': role.disqualified_by})
LOGGER.info('Getting data from Aardvark for account {}'.format(account_number))
aardvark_data = _get_aardvark_data(config['aardvark_api_location'], account_number=account_number)
LOGGER.info('Updating roles with Aardvark data in account {}'.format(account_number))
for role in roles:
try:
role.aa_data = aardvark_data[role.arn]
except KeyError:
LOGGER.warning('Aardvark data not found for role: {} ({})'.format(role.role_id, role.role_name))
else:
set_role_data(dynamo_table, role.role_id, {'AAData': role.aa_data})
LOGGER.info('Calculating repoable permissions and services for account {}'.format(account_number))
roledata._calculate_repo_scores(roles, config['filter_config']['AgeFilter']['minimum_age'], hooks)
for role in roles:
LOGGER.debug('Role {} in account {} has\nrepoable permissions: {}\nrepoable services:'.format(
role.role_name, account_number, role.repoable_permissions, role.repoable_services
))
set_role_data(dynamo_table, role.role_id, {'TotalPermissions': role.total_permissions,
'RepoablePermissions': role.repoable_permissions,
'RepoableServices': role.repoable_services})
LOGGER.info('Updating stats in account {}'.format(account_number))
roledata.update_stats(dynamo_table, roles, source='Scan')