def process_arguments(parsed_args: Namespace):
"""Given a namespace object generated from parsing args, perform the appropriate tasks. Returns an int
matching expectations set by /usr/include/sysexits.h for command-line utilities."""
if parsed_args.account is None:
session = botocore_tools.get_session(parsed_args.profile)
else:
session = None
graph = graph_actions.get_existing_graph(session, parsed_args.account)
logger.debug('Querying against graph {}'.format(
graph.metadata['account_id']))
# process condition args to generate input dict
conditions = {}
if parsed_args.condition is not None:
for arg in parsed_args.condition:
# split on equals-sign (=), assume first instance separates the key and value
components = arg.split('=')
if len(components) < 2:
print('Format for condition args not matched: <key>=<value>')
return 64
key = components[0]
value = '='.join(components[1:])
conditions.update({key: value})
if parsed_args.with_resource_policy:
resource_policy = query_utils.pull_cached_resource_policy_by_arn(
graph.policies, parsed_args.resource)
elif parsed_args.resource_policy_text:
resource_policy = json.loads(parsed_args.resource_policy_text)
else:
resource_policy = None
if parsed_args.scps:
if 'org-id' in graph.metadata and 'org-path' in graph.metadata:
org_tree_path = os.path.join(get_storage_root(),
graph.metadata['org-id'])
org_tree = OrganizationTree.create_from_dir(org_tree_path)
scps = query_orgs.produce_scp_list(graph, org_tree)
else:
raise ValueError(
'Graph for account {} does not have an associated OrganizationTree mapped (need to run '
'`pmapper orgs create/update` to get that.')
else:
scps = None
query_actions.argquery(graph, parsed_args.principal, parsed_args.action,
parsed_args.resource, conditions,
parsed_args.preset, parsed_args.skip_admin,
resource_policy, parsed_args.resource_owner,
parsed_args.include_unauthorized,
parsed_args.session_policy, scps)
return 0
def process_arguments(parsed_args: Namespace):
"""Given a namespace object generated from parsing args, perform the appropriate tasks. Returns an int
matching expectations set by /usr/include/sysexits.h for command-line utilities."""
if parsed_args.account is None:
session = botocore_tools.get_session(parsed_args.profile)
else:
session = None
graph = graph_actions.get_existing_graph(session, parsed_args.account)
logger.debug('Querying against graph {}'.format(graph.metadata['account_id']))
if parsed_args.with_resource_policy:
resource_policy = query_utils.pull_cached_resource_policy_by_arn(
graph,
arn=None,
query=parsed_args.query
)
elif parsed_args.resource_policy_text:
resource_policy = json.loads(parsed_args.resource_policy_text)
else:
resource_policy = None
resource_owner = parsed_args.resource_owner
if resource_policy is not None:
if resource_owner is None:
if arns.get_service(resource_policy.arn) == 's3':
raise ValueError('Must supply resource owner (--resource-owner) when including S3 bucket policies '
'in a query')
else:
resource_owner = arns.get_account_id(resource_policy.arn)
if isinstance(resource_policy, Policy):
resource_policy = resource_policy.policy_doc
if parsed_args.scps:
if 'org-id' in graph.metadata and 'org-path' in graph.metadata:
org_tree_path = os.path.join(get_storage_root(), graph.metadata['org-id'])
org_tree = OrganizationTree.create_from_dir(org_tree_path)
scps = query_orgs.produce_scp_list(graph, org_tree)
else:
raise ValueError('Graph for account {} does not have an associated OrganizationTree mapped (need to run '
'`pmapper orgs create/update` to get that.')
else:
scps = None
query_actions.query_response(
graph, parsed_args.query, parsed_args.skip_admin, resource_policy, resource_owner,
parsed_args.include_unauthorized, parsed_args.session_policy, scps
)
return 0
def get_organizations_data(
session: botocore.session.Session) -> OrganizationTree:
"""Given a botocore Session object, generate an OrganizationTree object. This throws a RuntimeError if the session
is for an Account that is not able to gather Organizations data, along with the reason why.
The edge_list field of the OrganizationTree object is not populated. """
# grab account data
stsclient = session.create_client('sts')
account_data = stsclient.get_caller_identity()
# try to grab org data, raising RuntimeError if appropriate
try:
orgsclient = session.create_client('organizations')
organization_data = orgsclient.describe_organization()
except botocore.exceptions.ClientError as ex:
if 'AccessDeniedException' in str(ex):
raise RuntimeError(
'Encountered a permission error. Either the current principal ({}) is not authorized to '
'interact with AWS Organizations, or the current account ({}) is not the '
'management account'.format(account_data['Arn'],
account_data['Account']))
else:
raise ex
# compose the OrganizationTree object
logger.info(
'Generating data for organization {} through management account {}'.
format(organization_data['Organization']['Id'],
organization_data['Organization']['MasterAccountId']))
result = OrganizationTree(
organization_data['Organization']['Id'],
organization_data['Organization']['MasterAccountId'],
None, # fill in `root_ous` later
None, # get SCPs later
None, # get account list later
[], # caller is responsible for creating and setting the edge list
{'pmapper_version': principalmapper.__version__})
scp_list = []
root_ous = []
account_ids = []
# get root IDs to start
logger.info('Going through roots of organization')
root_ids_and_names = []
list_roots_paginator = orgsclient.get_paginator('list_roots')
for page in list_roots_paginator.paginate():
root_ids_and_names.extend([(x['Id'], x['Name'])
for x in page['Roots']])
def _get_scps_for_target(target_id: str) -> List[Policy]:
"""This method takes an ID for a target (root, OU, or account), then composes and returns a list of Policy
objects for that target."""
scps_result = []
policy_name_arn_list = []
list_policies_paginator = orgsclient.get_paginator(
'list_policies_for_target')
for lpp_page in list_policies_paginator.paginate(
TargetId=target_id, Filter='SERVICE_CONTROL_POLICY'):
for policy in lpp_page['Policies']:
policy_name_arn_list.append((policy['Name'], policy['Arn']))
for name_arn_pair in policy_name_arn_list:
policy_name, policy_arn = name_arn_pair
desc_policy_resp = orgsclient.describe_policy(
PolicyId=policy_arn.split('/')[-1])
scps_result.append(
Policy(policy_arn, policy_name,
json.loads(desc_policy_resp['Policy']['Content'])))
logger.debug('SCPs of {}: {}'.format(target_id,
[x.arn for x in scps_result]))
scp_list.extend(scps_result)
return scps_result
def _get_tags_for_target(target_id: str) -> dict:
"""This method takes an ID for a target (root/OU/account) then composes and returns a dictionary for the
tags for that target"""
target_tags = {}
list_tags_paginator = orgsclient.get_paginator(
'list_tags_for_resource')
for ltp_page in list_tags_paginator.paginate(ResourceId=target_id):
for tag in ltp_page['Tags']:
target_tags[tag['Key']] = tag['Value']
logger.debug('Tags for {}: {}'.format(target_id, target_tags))
return target_tags
# for each root, recursively grab child OUs while filling out OrganizationNode/OrganizationAccount objects
# need to get tags, SCPs too
def _compose_ou(parent_id: str, parent_name: str) -> OrganizationNode:
"""This method takes an OU's ID and Name to compose and return an OrganizationNode object for that OU. This
grabs the accounts in the OU, tags for the OU, SCPs for the OU, and then gets the child OUs to recursively
compose those OrganizationNode objects."""
logger.info('Composing data for "{}" ({})'.format(
parent_name, parent_id))
# Get tags for the OU
ou_tags = _get_tags_for_target(parent_id)
# Get SCPs for the OU
ou_scps = _get_scps_for_target(parent_id)
# Get accounts under the OU
org_account_objs = [] # type: List[OrganizationAccount]
list_accounts_paginator = orgsclient.get_paginator(
'list_accounts_for_parent')
ou_child_account_list = []
for lap_page in list_accounts_paginator.paginate(ParentId=parent_id):
for child_account_data in lap_page['Accounts']:
ou_child_account_list.append(child_account_data['Id'])
logger.debug('Accounts: {}'.format(ou_child_account_list))
account_ids.extend(ou_child_account_list)
for ou_child_account_id in ou_child_account_list:
child_account_tags = _get_tags_for_target(ou_child_account_id)
child_account_scps = _get_scps_for_target(ou_child_account_id)
org_account_objs.append(
OrganizationAccount(ou_child_account_id, child_account_scps,
child_account_tags))
# get child OUs (pairs of Ids and Names)
child_ou_ids = []
list_children_paginator = orgsclient.get_paginator('list_children')
for lcp_page in list_children_paginator.paginate(
ParentId=parent_id, ChildType='ORGANIZATIONAL_UNIT'):
for child in lcp_page['Children']:
child_ou_ids.append(child['Id'])
child_ous = [] # type: List[OrganizationNode]
for child_ou_id in child_ou_ids:
desc_ou_resp = orgsclient.describe_organizational_unit(
OrganizationalUnitId=child_ou_id)
child_ous.append(
_compose_ou(child_ou_id,
desc_ou_resp['OrganizationalUnit']['Name']))
return OrganizationNode(parent_id, parent_name, org_account_objs,
child_ous, ou_scps, ou_tags)
for root_id_and_name in root_ids_and_names:
root_ou_id, root_ou_name = root_id_and_name
root_ous.append(_compose_ou(root_ou_id, root_ou_name))
# apply root OUs to result
result.root_ous = root_ous
# apply collected SCPs to result
filtered_scp_list = []
filtered_arns = []
for scp in scp_list:
if scp.arn in filtered_arns:
continue
filtered_scp_list.append(scp)
filtered_arns.append(scp.arn)
result.all_scps = filtered_scp_list
# apply collected account IDs to result
result.accounts = account_ids
return result
def process_arguments(parsed_args: Namespace):
"""Given a namespace object generated from parsing args, perform the appropriate tasks. Returns an int
matching expectations set by /usr/include/sysexits.h for command-line utilities."""
# new args for handling AWS Organizations
if parsed_args.picked_orgs_cmd == 'create':
logger.debug('Called create subcommand for organizations')
# filter the args first
if parsed_args.account is not None:
print(
'Cannot specify offline-mode param `--account` when calling `pmapper graph org_create`. If you have '
'credentials for a specific account to graph, you can use those credentials similar to how the '
'AWS CLI works (environment variables, profiles, EC2 instance metadata). In the case of using '
'a profile, use the `--profile [PROFILE]` argument before specifying the `graph` subcommand.'
)
return 64
# get the botocore session and go to work creating the OrganizationTree obj
session = botocore_tools.get_session(parsed_args.profile)
org_tree = get_organizations_data(session)
logger.info('Generated initial organization data for {}'.format(
org_tree.org_id))
# create the account -> OU path map and apply to all accounts (same as org_update operation)
account_ou_map = _map_account_ou_paths(org_tree)
logger.debug('account_ou_map: {}'.format(account_ou_map))
_update_accounts_with_ou_path_map(org_tree.org_id, account_ou_map,
get_storage_root())
logger.info(
'Updated currently stored Graphs with applicable AWS Organizations data'
)
# create and cache a list of edges between all the accounts we have data for
edge_list = []
graph_objs = []
for account in org_tree.accounts:
try:
potential_path = os.path.join(get_storage_root(), account)
logger.debug(
'Trying to load a Graph from {}'.format(potential_path))
graph_obj = Graph.create_graph_from_local_disk(potential_path)
graph_objs.append(graph_obj)
except Exception as ex:
logger.warning(
'Unable to load a Graph object for account {}, possibly because it is not mapped yet. '
'Please map all accounts and then update the Organization Tree '
'(`pmapper graph org_update`).'.format(account))
logger.debug(str(ex))
for graph_obj_a in graph_objs:
for graph_obj_b in graph_objs:
if graph_obj_a == graph_obj_b:
continue
graph_a_scps = produce_scp_list(graph_obj_a, org_tree)
graph_b_scps = produce_scp_list(graph_obj_b, org_tree)
edge_list.extend(
get_edges_between_graphs(graph_obj_a, graph_obj_b,
graph_a_scps, graph_b_scps))
org_tree.edge_list = edge_list
logger.info('Compiled cross-account edges')
org_tree.save_organization_to_disk(
os.path.join(get_storage_root(), org_tree.org_id))
logger.info('Stored organization data to disk')
elif parsed_args.picked_orgs_cmd == 'update':
# pull the existing data from disk
org_filepath = os.path.join(get_storage_root(), parsed_args.org)
org_tree = OrganizationTree.create_from_dir(org_filepath)
# create the account -> OU path map and apply to all accounts
account_ou_map = _map_account_ou_paths(org_tree)
logger.debug('account_ou_map: {}'.format(account_ou_map))
_update_accounts_with_ou_path_map(org_tree.org_id, account_ou_map,
get_storage_root())
logger.info(
'Updated currently stored Graphs with applicable AWS Organizations data'
)
# create and cache a list of edges between all the accounts we have data for
edge_list = []
graph_objs = []
for account in org_tree.accounts:
try:
potential_path = os.path.join(get_storage_root(), account)
logger.debug(
'Trying to load a Graph from {}'.format(potential_path))
graph_obj = Graph.create_graph_from_local_disk(potential_path)
graph_objs.append(graph_obj)
except Exception as ex:
logger.warning(
'Unable to load a Graph object for account {}, possibly because it is not mapped yet. '
'Please map all accounts and then update the Organization Tree '
'(`pmapper graph org_update`).'.format(account))
logger.debug(str(ex))
for graph_obj_a in graph_objs:
for graph_obj_b in graph_objs:
if graph_obj_a == graph_obj_b:
continue
graph_a_scps = produce_scp_list(graph_obj_a, org_tree)
graph_b_scps = produce_scp_list(graph_obj_b, org_tree)
edge_list.extend(
get_edges_between_graphs(graph_obj_a, graph_obj_b,
graph_a_scps, graph_b_scps))
org_tree.edge_list = edge_list
logger.info('Compiled cross-account edges')
org_tree.save_organization_to_disk(
os.path.join(get_storage_root(), org_tree.org_id))
logger.info('Stored organization data to disk')
elif parsed_args.picked_orgs_cmd == 'display':
# pull the existing data from disk
org_filepath = os.path.join(get_storage_root(), parsed_args.org)
org_tree = OrganizationTree.create_from_dir(org_filepath)
def _print_account(org_account: OrganizationAccount, indent_level: int,
inherited_scps: List[Policy]):
print('{} {}:'.format(' ' * indent_level, org_account.account_id))
print('{} Directly Attached SCPs: {}'.format(
' ' * indent_level, [x.name for x in org_account.scps]))
print('{} Inherited SCPs: {}'.format(
' ' * indent_level, [x.name for x in inherited_scps]))
def _walk_and_print_ou(org_node: OrganizationNode, indent_level: int,
inherited_scps: List[Policy]):
print('{}"{}" ({}):'.format(' ' * indent_level, org_node.ou_name,
org_node.ou_id))
print('{} Accounts:'.format(' ' * indent_level))
for o_account in org_node.accounts:
_print_account(o_account, indent_level + 2, inherited_scps)
print('{} Directly Attached SCPs: {}'.format(
' ' * indent_level, [x.name for x in org_node.scps]))
print('{} Inherited SCPs: {}'.format(
' ' * indent_level, [x.name for x in inherited_scps]))
print('{} Child OUs:'.format(' ' * indent_level))
for child_node in org_node.child_nodes:
new_inherited_scps = inherited_scps.copy()
new_inherited_scps.extend(
[x for x in org_node.scps if x not in inherited_scps])
_walk_and_print_ou(child_node, indent_level + 4,
new_inherited_scps)
print('Organization {}:'.format(org_tree.org_id))
for root_ou in org_tree.root_ous:
_walk_and_print_ou(root_ou, 0, [])
elif parsed_args.picked_orgs_cmd == 'list':
print("Organization IDs:")
print("---")
storage_root = Path(get_storage_root())
account_id_pattern = re.compile(r'o-\w+')
for direct in storage_root.iterdir():
if account_id_pattern.search(str(direct)) is not None:
metadata_file = direct.joinpath(Path('metadata.json'))
with open(str(metadata_file)) as fd:
version = json.load(fd)['pmapper_version']
print("{} (PMapper Version {})".format(direct.name, version))
return 0
def process_arguments(parsed_args: Namespace):
"""Given a namespace object generated from parsing args, perform the appropriate tasks. Returns an int
matching expectations set by /usr/include/sysexits.h for command-line utilities."""
if parsed_args.picked_graph_cmd == 'create':
logger.debug('Called create subcommand of graph')
# filter the args first
if parsed_args.account is not None:
print(
'Cannot specify offline-mode param `--account` when calling `pmapper graph create`. If you have '
'credentials for a specific account to graph, you can use those credentials similar to how the '
'AWS CLI works (environment variables, profiles, EC2 instance metadata). In the case of using '
'a profile, use the `--profile [PROFILE]` argument before specifying the `graph` subcommand.'
)
return 64
service_list_base = list(checker_map.keys())
if parsed_args.include_services is not None:
service_list = [
x for x in service_list_base
if x in parsed_args.include_services
]
elif parsed_args.exclude_services is not None:
service_list = [
x for x in service_list_base
if x not in parsed_args.exclude_services
]
else:
service_list = service_list_base
logger.debug(
'Service list after processing args: {}'.format(service_list))
# need to know account ID to search potential SCPs
if parsed_args.localstack_endpoint is not None:
session = botocore_tools.get_session(
parsed_args.profile,
{'endpoint_url': parsed_args.localstack_endpoint})
else:
session = botocore_tools.get_session(parsed_args.profile)
scps = None
if not parsed_args.ignore_orgs:
if parsed_args.localstack_endpoint is not None:
stsclient = session.create_client(
'sts', endpoint_url=parsed_args.localstack_endpoint)
else:
stsclient = session.create_client('sts')
caller_identity = stsclient.get_caller_identity()
caller_account = caller_identity['Account']
logger.debug("Caller Identity: {}".format(caller_identity))
org_tree_search_dir = Path(get_storage_root())
org_id_pattern = re.compile(r'/o-\w+')
for subdir in org_tree_search_dir.iterdir():
if org_id_pattern.search(str(subdir)) is not None:
logger.debug(
'Checking {} to see if account {} is a member'.format(
str(subdir), caller_account))
org_tree = OrganizationTree.create_from_dir(str(subdir))
if caller_account in org_tree.accounts:
logger.info(
'Account {} is a member of Organization {}'.format(
caller_account, org_tree.org_id))
if caller_account == org_tree.management_account_id:
logger.info(
'Account {} is the management account, SCPs do not apply'
.format(caller_account))
else:
logger.info(
'Identifying and applying SCPs for the graphing process'
)
scps = query_orgs.produce_scp_list_by_account_id(
caller_account, org_tree)
break
if parsed_args.localstack_endpoint is not None:
full_service_list = ('autoscaling', 'cloudformation', 'codebuild',
'ec2', 'iam', 'kms', 'lambda', 'sagemaker',
's3', 'ssm', 'secretsmanager', 'sns', 'sts',
'sqs')
client_args_map = {
x: {
'endpoint_url': parsed_args.localstack_endpoint
}
for x in full_service_list
}
else:
client_args_map = None
graph = graph_actions.create_new_graph(session, service_list,
parsed_args.include_regions,
parsed_args.exclude_regions,
scps, client_args_map)
graph_actions.print_graph_data(graph)
graph.store_graph_as_json(
os.path.join(get_storage_root(), graph.metadata['account_id']))
elif parsed_args.picked_graph_cmd == 'display':
if parsed_args.account is None:
session = botocore_tools.get_session(parsed_args.profile)
else:
session = None
graph = graph_actions.get_existing_graph(session, parsed_args.account)
graph_actions.print_graph_data(graph)
elif parsed_args.picked_graph_cmd == 'list':
print("Account IDs:")
print("---")
storage_root = Path(get_storage_root())
account_id_pattern = re.compile(r'\d{12}')
for direct in storage_root.iterdir():
if account_id_pattern.search(str(direct)) is not None:
metadata_file = direct.joinpath(Path('metadata.json'))
with open(str(metadata_file)) as fd:
account_metadata = json.load(fd)
version = account_metadata['pmapper_version']
print("{} (PMapper Version {})".format(direct.name, version))
return 0