def test_with_under_populated_global_region_whitelist(self):
sample_data_filepath = "tests/data/aws_service_region_mapping/20210329202700.json"
with open(sample_data_filepath, "r") as fp:
region_services_json = json.load(fp)
with unittest.mock.patch(
"altimeter.aws.resource_service_region_mapping.get_aws_service_region_mapping_json"
) as mock_get_aws_service_region_mapping_json:
mock_get_aws_service_region_mapping_json.return_value = region_services_json
mapping_repo = build_aws_resource_region_mapping_repo(
global_region_whitelist=("us-east-2", "us-west-1",
"us-west-2"),
preferred_account_scan_regions=("us-east-1", ),
services_regions_json_url="https://mock_url",
)
for resource_spec_class in ALL_RESOURCE_SPEC_CLASSES:
if resource_spec_class.scan_granularity == ScanGranularity.ACCOUNT:
with self.assertRaises(NoRegionsFoundForResource):
mapping_repo.get_regions(
resource_spec_class=resource_spec_class,
region_whitelist=())
else:
scan_regions = mapping_repo.get_regions(
resource_spec_class=resource_spec_class,
region_whitelist=())
self.assertGreaterEqual(len(scan_regions), 1)
self.assertLessEqual(len(scan_regions), 4)
def aws2neptune_lpg(scan_id: str, config: AWSConfig,
muxer: AWSScanMuxer) -> None:
"""Scan AWS resources to json, convert to RDF and load into Neptune
if config.neptune is defined"""
artifact_reader = ArtifactReader.from_artifact_path(config.artifact_path)
artifact_writer = ArtifactWriter.from_artifact_path(
artifact_path=config.artifact_path, scan_id=scan_id)
aws_resource_region_mapping_repo = build_aws_resource_region_mapping_repo(
global_region_whitelist=config.scan.regions,
preferred_account_scan_regions=config.scan.
preferred_account_scan_regions,
services_regions_json_url=config.services_regions_json_url,
)
logger.info(
AWSLogEvents.ScanConfigured,
config=str(config),
reader=str(artifact_reader.__class__),
writer=str(artifact_writer.__class__),
)
print("Beginning AWS Account Scan")
_, graph_set = run_scan(
muxer=muxer,
config=config,
aws_resource_region_mapping_repo=aws_resource_region_mapping_repo,
artifact_writer=artifact_writer,
artifact_reader=artifact_reader,
)
print("AWS Account Scan Complete. Beginning write to Amazon Neptune.")
logger.info(LogEvent.NeptuneGremlinWriteStart)
graph = graph_set.to_neptune_lpg(scan_id)
if config.neptune is None:
raise Exception(
"Can not load to Neptune because config.neptune is empty.")
endpoint = NeptuneEndpoint(
host=config.neptune.host,
port=config.neptune.port,
region=config.neptune.region,
ssl=bool(config.neptune.ssl),
auth_mode=str(config.neptune.auth_mode),
)
neptune_client = AltimeterNeptuneClient(max_age_min=1440,
neptune_endpoint=endpoint)
neptune_client.write_to_neptune_lpg(graph, scan_id)
logger.info(LogEvent.NeptuneGremlinWriteEnd)
print("Write to Amazon Neptune Complete")
def aws2n(scan_id: str, config: AWSConfig, muxer: AWSScanMuxer,
load_neptune: bool) -> AWS2NResult:
"""Scan AWS resources to json, convert to RDF and load into Neptune
if config.neptune is defined"""
artifact_reader = ArtifactReader.from_artifact_path(config.artifact_path)
artifact_writer = ArtifactWriter.from_artifact_path(
artifact_path=config.artifact_path, scan_id=scan_id)
aws_resource_region_mapping_repo = build_aws_resource_region_mapping_repo(
global_region_whitelist=config.scan.regions,
preferred_account_scan_regions=config.scan.
preferred_account_scan_regions,
services_regions_json_url=config.services_regions_json_url,
)
logger = Logger()
logger.info(
AWSLogEvents.ScanConfigured,
config=str(config),
reader=str(artifact_reader.__class__),
writer=str(artifact_writer.__class__),
)
scan_manifest, graph_set = run_scan(
muxer=muxer,
config=config,
aws_resource_region_mapping_repo=aws_resource_region_mapping_repo,
artifact_writer=artifact_writer,
artifact_reader=artifact_reader,
)
json_path = scan_manifest.master_artifact
compression = GZIP if config.artifact_compressed else None
rdf_path = artifact_writer.write_graph_set(name="master",
graph_set=graph_set,
compression=compression)
graph_metadata = None
if load_neptune:
if config.neptune is None:
raise Exception(
"Can not load to Neptune because config.neptune is empty.")
endpoint = NeptuneEndpoint(host=config.neptune.host,
port=config.neptune.port,
region=config.neptune.region)
neptune_client = AltimeterNeptuneClient(max_age_min=1440,
neptune_endpoint=endpoint)
rdf_bucket, rdf_key = parse_s3_uri(rdf_path)
if rdf_key is None:
raise Exception(f"Invalid rdf s3 path {rdf_path}")
graph_metadata = neptune_client.load_graph(
bucket=rdf_bucket,
key=rdf_key,
load_iam_role_arn=str(config.neptune.iam_role_arn))
logger.info(event=LogEvent.GraphLoadedSNSNotificationStart)
sns_client = boto3.client("sns")
message_dict = {
"uri": graph_metadata.uri,
"name": graph_metadata.name,
"version": graph_metadata.version,
"start_time": graph_metadata.start_time,
"end_time": graph_metadata.end_time,
"neptune_endpoint": endpoint.get_endpoint_str(),
}
message_dict["default"] = json.dumps(message_dict)
sns_client.publish(
TopicArn=config.neptune.graph_load_sns_topic_arn,
MessageStructure="json",
Message=json.dumps(message_dict),
)
logger.info(event=LogEvent.GraphLoadedSNSNotificationEnd)
return AWS2NResult(rdf_path=rdf_path,
graph_metadata=graph_metadata,
json_path=json_path)
