def _load_s3_acls(session, acls, aws_account_id, update_tag):
"""
Ingest S3 ACL into neo4j.
"""
ingest_acls = """
UNWIND {acls} AS acl
MERGE (a:S3Acl{id: acl.id})
ON CREATE SET a.firstseen = timestamp(), a.owner = acl.owner, a.ownerid = acl.ownerid, a.type = acl.type,
a.displayname = acl.displayname, a.granteeid = acl.granteeid, a.uri = acl.uri, a.permission = acl.permission
SET a.lastupdated = {UpdateTag}
WITH a,acl MATCH (s3:S3Bucket{name: acl.bucket})
MERGE (a)-[r:APPLIES_TO]->(s3)
ON CREATE SET r.firstseen = timestamp()
SET r.lastupdated = {UpdateTag}
"""
session.run(
ingest_acls,
acls=acls,
UpdateTag=update_tag
)
# implement the acl permission
# https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#permissions
run_analysis_job(
'aws_s3acl_analysis.json',
session,
{'AWS_ID': aws_account_id}
)
def test_ec2_iaminstanceprofiles(neo4j_session):
"""
Ensure that EC2Instances are attached to the IAM Roles that they can assume due to their IAM instance profiles
"""
neo4j_session.run(
"""
MERGE (aws:AWSAccount{id: {aws_account_id}})
ON CREATE SET aws.firstseen = timestamp()
SET aws.lastupdated = {aws_update_tag}
""",
aws_account_id=TEST_ACCOUNT_ID,
aws_update_tag=TEST_UPDATE_TAG,
)
data_instances = tests.data.aws.ec2.instances.DESCRIBE_INSTANCES[
'Reservations']
data_iam = tests.data.aws.iam.INSTACE['Roles']
cartography.intel.aws.ec2.instances.load_ec2_instances(
neo4j_session,
data_instances,
TEST_REGION,
TEST_ACCOUNT_ID,
TEST_UPDATE_TAG,
)
cartography.intel.aws.iam.load_roles(
neo4j_session,
data_iam,
TEST_ACCOUNT_ID,
TEST_UPDATE_TAG,
)
common_job_parameters = {
"UPDATE_TAG": TEST_UPDATE_TAG,
}
run_analysis_job(
'aws_ec2_iaminstanceprofile.json',
neo4j_session,
common_job_parameters,
)
expected_nodes = {
('arn:aws:iam::000000000000:role/SERVICE_NAME_2', 'i-02'),
('arn:aws:iam::000000000000:role/ANOTHER_SERVICE_NAME', 'i-03'),
('arn:aws:iam::000000000000:role/ANOTHER_SERVICE_NAME', 'i-04'),
}
nodes = neo4j_session.run(
"""
MATCH (i:EC2Instance)-[:STS_ASSUMEROLE_ALLOW]->(r:AWSRole) return r.arn, i.id
""", )
actual_nodes = {(
n['r.arn'],
n['i.id'],
)
for n in nodes}
assert actual_nodes == expected_nodes
def test_run_analysis_job_custom_package(mocker):
mocker.patch('cartography.util.GraphJob')
read_text_mock = mocker.patch('cartography.util.read_text')
util.run_analysis_job('test.json',
mocker.Mock(),
mocker.Mock(),
package='a.b.c')
read_text_mock.assert_called_once_with('a.b.c', 'test.json')
def start_aws_ingestion(session, config):
common_job_parameters = {
"UPDATE_TAG": config.update_tag,
}
try:
default_boto3_session = boto3.Session()
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
logger.debug("Error occurred calling boto3.Session().", exc_info=True)
logger.error(
(
"Unable to initialize the default AWS session, an error occurred: %s. Make sure your AWS credentials "
"are configured correctly, your AWS config file is valid, and your credentials have the SecurityAudit "
"policy attached."
),
e
)
return
if config.aws_sync_all_profiles:
aws_accounts = organizations.get_aws_accounts_from_botocore_config(default_boto3_session)
else:
aws_accounts = organizations.get_aws_account_default(default_boto3_session)
if not aws_accounts:
logger.warning(
"No valid AWS credentials could be found. No AWS accounts can be synced. Exiting AWS sync stage."
)
return
if len(list(aws_accounts.values())) != len(set(aws_accounts.values())):
logger.warning(
(
"There are duplicate AWS accounts in your AWS configuration. It is strongly recommended that you run "
"cartography with an AWS configuration which has exactly one profile for each AWS account you want to "
"sync. Doing otherwise will result in undefined and untested behavior."
)
)
try:
regions = ec2.get_ec2_regions(default_boto3_session)
except botocore.exceptions.ClientError as e:
logger.debug("Error occurred getting EC2 regions.", exc_info=True)
logger.error(
(
"Failed to retrieve AWS region list, an error occurred: %s. The AWS sync cannot run without a valid "
"region list."
),
e
)
return
_sync_multiple_accounts(session, aws_accounts, regions, config.update_tag, common_job_parameters)
run_analysis_job(
'aws_ec2_asset_exposure.json',
session,
common_job_parameters
)
def _sync_one_account(
neo4j_session: neo4j.Session,
boto3_session: boto3.session.Session,
current_aws_account_id: str,
update_tag: int,
common_job_parameters: Dict[str, Any],
regions: List[str] = [],
aws_requested_syncs: Iterable[str] = RESOURCE_FUNCTIONS.keys(),
) -> None:
if not regions:
regions = _autodiscover_account_regions(boto3_session,
current_aws_account_id)
sync_args = _build_aws_sync_kwargs(
neo4j_session,
boto3_session,
regions,
current_aws_account_id,
update_tag,
common_job_parameters,
)
for func_name in aws_requested_syncs:
if func_name in RESOURCE_FUNCTIONS:
# Skip permission relationships and tags for now because they rely on data already being in the graph
if func_name not in [
'permission_relationships', 'resourcegroupstaggingapi'
]:
RESOURCE_FUNCTIONS[func_name](**sync_args)
else:
continue
else:
raise ValueError(
f'AWS sync function "{func_name}" was specified but does not exist. Did you misspell it?'
)
# NOTE clean up all DNS records, regardless of which job created them
run_cleanup_job('aws_account_dns_cleanup.json', neo4j_session,
common_job_parameters)
# MAP IAM permissions
if 'permission_relationships' in aws_requested_syncs:
RESOURCE_FUNCTIONS['permission_relationships'](**sync_args)
# AWS Tags - Must always be last.
if 'resourcegroupstaggingapi' in aws_requested_syncs:
RESOURCE_FUNCTIONS['resourcegroupstaggingapi'](**sync_args)
run_analysis_job(
'aws_ec2_iaminstanceprofile.json',
neo4j_session,
common_job_parameters,
)
def start_gcp_ingestion(neo4j_session, config):
"""
Starts the GCP ingestion process by initializing Google Application Default Credentials, creating the necessary
resource objects, listing all GCP organizations and projects available to the GCP identity, and supplying that
context to all intel modules.
:param neo4j_session: The Neo4j session
:param config: A `cartography.config` object
:return: Nothing
"""
common_job_parameters = {
"UPDATE_TAG": config.update_tag,
}
try:
# Explicitly use Application Default Credentials.
# See https://oauth2client.readthedocs.io/en/latest/source/
# oauth2client.client.html#oauth2client.client.OAuth2Credentials
credentials = GoogleCredentials.get_application_default()
except ApplicationDefaultCredentialsError as e:
logger.debug(
"Error occurred calling GoogleCredentials.get_application_default().",
exc_info=True)
logger.error(
("Unable to initialize Google Compute Platform creds. If you don't have GCP data or don't want to load "
"GCP data then you can ignore this message. Otherwise, the error code is: %s "
"Make sure your GCP credentials are configured correctly, your credentials file (if any) is valid, and "
"that the identity you are authenticating to has the securityReviewer role attached."
),
e,
)
return
resources = _initialize_resources(credentials)
# If we don't have perms to pull Orgs or Folders from GCP, we will skip safely
crm.sync_gcp_organizations(neo4j_session, resources.crm_v1,
config.update_tag, common_job_parameters)
crm.sync_gcp_folders(neo4j_session, resources.crm_v2, config.update_tag,
common_job_parameters)
projects = crm.get_gcp_projects(resources.crm_v1)
_sync_multiple_projects(neo4j_session, resources, projects,
config.update_tag, common_job_parameters)
run_analysis_job(
'gcp_compute_asset_inet_exposure.json',
neo4j_session,
common_job_parameters,
)
def start_aws_ingestion(neo4j_session: neo4j.Session, config: Config) -> None:
common_job_parameters = {
"UPDATE_TAG": config.update_tag,
"permission_relationships_file": config.permission_relationships_file,
}
try:
boto3_session = boto3.Session()
except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e:
logger.debug("Error occurred calling boto3.Session().", exc_info=True)
logger.error(
(
"Unable to initialize the default AWS session, an error occurred: %s. Make sure your AWS credentials "
"are configured correctly, your AWS config file is valid, and your credentials have the SecurityAudit "
"policy attached."
),
e,
)
return
if config.aws_sync_all_profiles:
aws_accounts = organizations.get_aws_accounts_from_botocore_config(boto3_session)
else:
aws_accounts = organizations.get_aws_account_default(boto3_session)
if not aws_accounts:
logger.warning(
"No valid AWS credentials could be found. No AWS accounts can be synced. Exiting AWS sync stage.",
)
return
if len(list(aws_accounts.values())) != len(set(aws_accounts.values())):
logger.warning(
(
"There are duplicate AWS accounts in your AWS configuration. It is strongly recommended that you run "
"cartography with an AWS configuration which has exactly one profile for each AWS account you want to "
"sync. Doing otherwise will result in undefined and untested behavior."
),
)
requested_syncs: List[str] = list(RESOURCE_FUNCTIONS.keys())
if config.aws_requested_syncs:
requested_syncs = parse_and_validate_aws_requested_syncs(config.aws_requested_syncs)
_sync_multiple_accounts(
neo4j_session,
aws_accounts,
config.update_tag,
common_job_parameters,
requested_syncs,
)
run_analysis_job(
'aws_ec2_asset_exposure.json',
neo4j_session,
common_job_parameters,
)
run_analysis_job(
'aws_ec2_keypair_analysis.json',
neo4j_session,
common_job_parameters,
)
run_analysis_job(
'aws_eks_asset_exposure.json',
neo4j_session,
common_job_parameters,
)
def test_run_analysis_job_default_package(mocker):
mocker.patch('cartography.util.GraphJob')
read_text_mock = mocker.patch('cartography.util.read_text')
util.run_analysis_job('test.json', mocker.Mock(), mocker.Mock())
read_text_mock.assert_called_once_with('cartography.data.jobs.analysis',
'test.json')
