def test_from_dict(self):
config_dict = {
'artifact_path': '/tmp/altimeter_single_account',
'pruner_max_age_min': 4320,
'graph_name': 'alti',
'access': {
'accessor': {
'multi_hop_accessors': [],
'credentials_cache': {
'cache': {}
}
}
},
'concurrency': {
'max_account_scan_threads': 1,
'max_accounts_per_thread': 1,
'max_svc_scan_threads': 64
},
'scan': {
'accounts': ('1234',),
'regions': (),
'scan_sub_accounts': False,
'preferred_account_scan_regions': (
'us-west-1',
'us-west-2',
'us-east-1',
'us-east-2'
),
'single_account_mode': False
},
'neptune': None
}
config = Config.from_dict(config_dict)
self.assertIsNone(config.neptune)
self.assertEqual(config.pruner_max_age_min, 4320)
def main(argv: Optional[List[str]] = None) -> int:
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("config", type=str, nargs="?")
args_ns = parser.parse_args(argv)
config = args_ns.config
if config is None:
config = os.environ.get("CONFIG_PATH")
if config is None:
print(
"config must be provided as a positional arg or env var 'CONFIG_PATH'"
)
return 1
config = Config.from_path(config)
scan_id = generate_scan_id()
muxer = LocalAWSScanMuxer(scan_id=scan_id, config=config)
result = aws2n(scan_id=scan_id,
config=config,
muxer=muxer,
load_neptune=False)
print(result.rdf_path)
return 0
def test_from_dict(self):
scan_config_dict = {
"accounts": ["123", "456"],
"regions": ["us-west-2", "us-west-1"],
"scan_sub_accounts": False,
"preferred_account_scan_regions": ["us-east-1", "us-west-2"],
"single_account_mode": False,
}
scan_config = ScanConfig.from_dict(scan_config_dict)
self.assertTupleEqual(scan_config.accounts, ("123", "456"))
self.assertTupleEqual(scan_config.regions, ("us-west-2", "us-west-1"))
self.assertEqual(scan_config.scan_sub_accounts, False)
self.assertTupleEqual(scan_config.preferred_account_scan_regions, ("us-east-1", "us-west-2"))
self.assertEqual(scan_config.single_account_mode, False)
config = Config.from_file("conf/single_account.toml")
def lambda_handler(event: Dict[str, Any], context: Any) -> None:
"""AWS Lambda Handler"""
root = logging.getLogger()
if root.handlers:
for handler in root.handlers:
root.removeHandler(handler)
account_scan_lambda_name = get_required_str_env_var(
"ACCOUNT_SCAN_LAMBDA_NAME")
account_scan_lambda_timeout = get_required_int_env_var(
"ACCOUNT_SCAN_LAMBDA_TIMEOUT")
config_path = get_required_str_env_var("CONFIG_PATH")
config = Config.from_path(path=config_path)
scan_id = generate_scan_id()
muxer = LambdaAWSScanMuxer(
scan_id=scan_id,
config=config,
account_scan_lambda_name=account_scan_lambda_name,
account_scan_lambda_timeout=account_scan_lambda_timeout,
)
aws2n(scan_id=scan_id, config=config, muxer=muxer, load_neptune=True)
def main(argv: Optional[List[str]] = None) -> int:
"""Main method for running a AWS to Neptune run"""
if argv is None:
argv = sys.argv[1:]
parser = argparse.ArgumentParser()
parser.add_argument("model", type=str, nargs="?")
parser.add_argument("config", type=str, nargs="?")
args_ns = parser.parse_args(argv)
if args_ns.model is None:
print("You need to specify the data model desired for this run (RDF or LPG).")
sys.exit()
if args_ns.model.lower() != "rdf" and args_ns.model.lower() != "lpg":
print("You need to specify the data model desired for this run (RDF or LPG).")
sys.exit()
config_path = (
"/home/ec2-user/graph_notebook_config.json" if args_ns.config is None else args_ns.config
)
with open(config_path) as f:
config = json.load(f)
current_account = boto3.client("sts").get_caller_identity().get("Account")
current_region = boto3.session.Session().region_name
config_dict = {
"artifact_path": "./altimeter_runs",
"pruner_max_age_min": 4320,
"graph_name": "alti",
"access": {"cache_creds": True},
"concurrency": {
"max_account_scan_threads": 1,
"max_accounts_per_thread": 1,
"max_svc_scan_threads": 64,
},
"scan": {
"accounts": [current_account],
"regions": [current_region],
"scan_sub_accounts": False,
"preferred_account_scan_regions": [current_region],
},
"neptune": {
"host": config["host"],
"port": int(config["port"]),
"auth_mode": config["auth_mode"],
"iam_credentials_provider_type": config["iam_credentials_provider_type"],
"ssl": config["ssl"],
"region": config["aws_region"],
"use_lpg": bool(args_ns.model.lower() == "lpg"),
},
}
config = Config.from_dict(config_dict)
scan_id = generate_scan_id()
muxer = LocalAWSScanMuxer(scan_id=scan_id, config=config)
if config.neptune:
if bool(config.neptune.use_lpg):
aws2neptune_lpg(scan_id=scan_id, config=config, muxer=muxer)
else:
aws2neptune_rdf(scan_id=scan_id, config=config, muxer=muxer)
else:
raise Exception("Can not load to Neptune because config.neptune is empty.")
return 0
def prune_graph(graph_pruner_config: GraphPrunerConfig) -> GraphPrunerResults:
config = Config.from_path(path=graph_pruner_config.config_path)
if config.neptune is None:
raise InvalidConfigException("Configuration missing neptune section.")
now = int(datetime.now().timestamp())
oldest_acceptable_graph_epoch = now - config.pruner_max_age_min * 60
endpoint = NeptuneEndpoint(
host=config.neptune.host, port=config.neptune.port, region=config.neptune.region
)
client = AltimeterNeptuneClient(
max_age_min=config.pruner_max_age_min, neptune_endpoint=endpoint
)
logger = Logger()
uncleared = []
pruned_graph_uris = []
skipped_graph_uris = []
logger.info(event=LogEvent.PruneNeptuneGraphsStart)
all_graph_metadatas = client.get_graph_metadatas(name=config.graph_name)
with logger.bind(neptune_endpoint=str(endpoint)):
for graph_metadata in all_graph_metadatas:
assert graph_metadata.name == config.graph_name
graph_epoch = graph_metadata.end_time
with logger.bind(graph_uri=graph_metadata.uri, graph_epoch=graph_epoch):
if graph_epoch < oldest_acceptable_graph_epoch:
logger.info(event=LogEvent.PruneNeptuneGraphStart)
try:
client.clear_registered_graph(
name=config.graph_name, uri=graph_metadata.uri
)
logger.info(event=LogEvent.PruneNeptuneGraphEnd)
pruned_graph_uris.append(graph_metadata.uri)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=f"Error pruning graph {graph_metadata.uri}: {ex}",
)
uncleared.append(graph_metadata.uri)
continue
else:
logger.info(event=LogEvent.PruneNeptuneGraphSkip)
skipped_graph_uris.append(graph_metadata.uri)
# now find orphaned graphs - these are in neptune but have no metadata
registered_graph_uris = [g_m.uri for g_m in all_graph_metadatas]
all_graph_uris = client.get_graph_uris(name=config.graph_name)
orphaned_graphs = set(all_graph_uris) - set(registered_graph_uris)
if orphaned_graphs:
for orphaned_graph_uri in orphaned_graphs:
with logger.bind(graph_uri=orphaned_graph_uri):
logger.info(event=LogEvent.PruneOrphanedNeptuneGraphStart)
try:
client.clear_graph_data(uri=orphaned_graph_uri)
logger.info(event=LogEvent.PruneOrphanedNeptuneGraphEnd)
pruned_graph_uris.append(orphaned_graph_uri)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=f"Error pruning graph {orphaned_graph_uri}: {ex}",
)
uncleared.append(orphaned_graph_uri)
continue
logger.info(event=LogEvent.PruneNeptuneGraphsEnd)
if uncleared:
msg = f"Errors were found pruning {uncleared}."
logger.error(event=LogEvent.PruneNeptuneGraphsError, msg=msg)
raise Exception(msg)
return GraphPrunerResults(
pruned_graph_uris=pruned_graph_uris, skipped_graph_uris=skipped_graph_uris,
)
def lambda_handler(event: Dict[str, Any], context: Any) -> None:
"""Entrypoint"""
root = logging.getLogger()
if root.handlers:
for handler in root.handlers:
root.removeHandler(handler)
config_path = get_required_str_env_var("CONFIG_PATH")
config = Config.from_path(path=config_path)
if config.neptune is None:
raise InvalidConfigException("Configuration missing neptune section.")
now = int(datetime.now().timestamp())
oldest_acceptable_graph_epoch = now - config.pruner_max_age_min * 60
endpoint = NeptuneEndpoint(host=config.neptune.host,
port=config.neptune.port,
region=config.neptune.region)
client = AltimeterNeptuneClient(max_age_min=config.pruner_max_age_min,
neptune_endpoint=endpoint)
logger = Logger()
uncleared = []
logger.info(event=LogEvent.PruneNeptuneGraphsStart)
all_graph_metadatas = client.get_graph_metadatas(name=config.graph_name)
with logger.bind(neptune_endpoint=str(endpoint)):
for graph_metadata in all_graph_metadatas:
assert graph_metadata.name == config.graph_name
graph_epoch = graph_metadata.end_time
with logger.bind(graph_uri=graph_metadata.uri,
graph_epoch=graph_epoch):
if graph_epoch < oldest_acceptable_graph_epoch:
logger.info(event=LogEvent.PruneNeptuneGraphStart)
try:
client.clear_registered_graph(name=config.graph_name,
uri=graph_metadata.uri)
logger.info(event=LogEvent.PruneNeptuneGraphEnd)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=
f"Error pruning graph {graph_metadata.uri}: {ex}",
)
uncleared.append(graph_metadata.uri)
continue
else:
logger.info(event=LogEvent.PruneNeptuneGraphSkip)
# now find orphaned graphs - these are in neptune but have no metadata
registered_graph_uris = [g_m.uri for g_m in all_graph_metadatas]
all_graph_uris = client.get_graph_uris(name=config.graph_name)
orphaned_graphs = set(all_graph_uris) - set(registered_graph_uris)
if orphaned_graphs:
for orphaned_graph_uri in orphaned_graphs:
with logger.bind(graph_uri=orphaned_graph_uri):
logger.info(event=LogEvent.PruneOrphanedNeptuneGraphStart)
try:
client.clear_graph_data(uri=orphaned_graph_uri)
logger.info(
event=LogEvent.PruneOrphanedNeptuneGraphEnd)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=
f"Error pruning graph {orphaned_graph_uri}: {ex}",
)
uncleared.append(orphaned_graph_uri)
continue
logger.info(event=LogEvent.PruneNeptuneGraphsEnd)
if uncleared:
msg = f"Errors were found pruning {uncleared}."
logger.error(event=LogEvent.PruneNeptuneGraphsError, msg=msg)
raise Exception(msg)