def lambda_handler(event, context):
host = get_required_lambda_env_var("NEPTUNE_HOST")
port = get_required_lambda_env_var("NEPTUNE_PORT")
region = get_required_lambda_env_var("NEPTUNE_REGION")
max_age_min = get_required_lambda_env_var("MAX_AGE_MIN")
graph_name = get_required_lambda_env_var("GRAPH_NAME")
try:
max_age_min = int(max_age_min)
except ValueError as ve:
raise Exception(f"env var MAX_AGE_MIN must be an int: {ve}")
now = int(datetime.now().timestamp())
oldest_acceptable_graph_epoch = now - max_age_min * 60
endpoint = NeptuneEndpoint(host=host, port=port, region=region)
client = AltimeterNeptuneClient(max_age_min=max_age_min,
neptune_endpoint=endpoint)
logger = Logger()
uncleared = []
# first prune metadata - if clears below are partial we want to make sure no clients
# consider this a valid graph still.
logger.info(event=LogEvent.PruneNeptuneMetadataGraphStart)
client.clear_old_graph_metadata(name=graph_name, max_age_min=max_age_min)
logger.info(event=LogEvent.PruneNeptuneMetadataGraphEnd)
# now clear actual graphs
with logger.bind(neptune_endpoint=endpoint):
logger.info(event=LogEvent.PruneNeptuneGraphsStart)
for graph_metadata in client.get_graph_metadatas(name=graph_name):
assert graph_metadata.name == graph_name
graph_epoch = graph_metadata.end_time
with logger.bind(graph_uri=graph_metadata.uri,
graph_epoch=graph_epoch):
if graph_epoch < oldest_acceptable_graph_epoch:
logger.info(event=LogEvent.PruneNeptuneGraphStart)
try:
client.clear_graph(graph_uri=graph_metadata.uri)
logger.info(event=LogEvent.PruneNeptuneGraphEnd)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=
f"Error pruning graph {graph_metadata.uri}: {ex}",
)
uncleared.append(graph_metadata.uri)
continue
else:
logger.info(event=LogEvent.PruneNeptuneGraphSkip)
logger.info(event=LogEvent.PruneNeptuneGraphsEnd)
if uncleared:
msg = f"Errors were found pruning {uncleared}."
logger.error(event=LogEvent.PruneNeptuneGraphsError, msg=msg)
raise Exception(msg)
def prune_graph(graph_pruner_config: GraphPrunerConfig) -> GraphPrunerResults:
config = Config.from_path(path=graph_pruner_config.config_path)
if config.neptune is None:
raise InvalidConfigException("Configuration missing neptune section.")
now = int(datetime.now().timestamp())
oldest_acceptable_graph_epoch = now - config.pruner_max_age_min * 60
endpoint = NeptuneEndpoint(
host=config.neptune.host, port=config.neptune.port, region=config.neptune.region
)
client = AltimeterNeptuneClient(
max_age_min=config.pruner_max_age_min, neptune_endpoint=endpoint
)
logger = Logger()
uncleared = []
pruned_graph_uris = []
skipped_graph_uris = []
logger.info(event=LogEvent.PruneNeptuneGraphsStart)
all_graph_metadatas = client.get_graph_metadatas(name=config.graph_name)
with logger.bind(neptune_endpoint=str(endpoint)):
for graph_metadata in all_graph_metadatas:
assert graph_metadata.name == config.graph_name
graph_epoch = graph_metadata.end_time
with logger.bind(graph_uri=graph_metadata.uri, graph_epoch=graph_epoch):
if graph_epoch < oldest_acceptable_graph_epoch:
logger.info(event=LogEvent.PruneNeptuneGraphStart)
try:
client.clear_registered_graph(
name=config.graph_name, uri=graph_metadata.uri
)
logger.info(event=LogEvent.PruneNeptuneGraphEnd)
pruned_graph_uris.append(graph_metadata.uri)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=f"Error pruning graph {graph_metadata.uri}: {ex}",
)
uncleared.append(graph_metadata.uri)
continue
else:
logger.info(event=LogEvent.PruneNeptuneGraphSkip)
skipped_graph_uris.append(graph_metadata.uri)
# now find orphaned graphs - these are in neptune but have no metadata
registered_graph_uris = [g_m.uri for g_m in all_graph_metadatas]
all_graph_uris = client.get_graph_uris(name=config.graph_name)
orphaned_graphs = set(all_graph_uris) - set(registered_graph_uris)
if orphaned_graphs:
for orphaned_graph_uri in orphaned_graphs:
with logger.bind(graph_uri=orphaned_graph_uri):
logger.info(event=LogEvent.PruneOrphanedNeptuneGraphStart)
try:
client.clear_graph_data(uri=orphaned_graph_uri)
logger.info(event=LogEvent.PruneOrphanedNeptuneGraphEnd)
pruned_graph_uris.append(orphaned_graph_uri)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=f"Error pruning graph {orphaned_graph_uri}: {ex}",
)
uncleared.append(orphaned_graph_uri)
continue
logger.info(event=LogEvent.PruneNeptuneGraphsEnd)
if uncleared:
msg = f"Errors were found pruning {uncleared}."
logger.error(event=LogEvent.PruneNeptuneGraphsError, msg=msg)
raise Exception(msg)
return GraphPrunerResults(
pruned_graph_uris=pruned_graph_uris, skipped_graph_uris=skipped_graph_uris,
)
def lambda_handler(event: Dict[str, Any], context: Any) -> None:
"""Entrypoint"""
root = logging.getLogger()
if root.handlers:
for handler in root.handlers:
root.removeHandler(handler)
config_path = get_required_str_env_var("CONFIG_PATH")
config = Config.from_path(path=config_path)
if config.neptune is None:
raise InvalidConfigException("Configuration missing neptune section.")
now = int(datetime.now().timestamp())
oldest_acceptable_graph_epoch = now - config.pruner_max_age_min * 60
endpoint = NeptuneEndpoint(host=config.neptune.host,
port=config.neptune.port,
region=config.neptune.region)
client = AltimeterNeptuneClient(max_age_min=config.pruner_max_age_min,
neptune_endpoint=endpoint)
logger = Logger()
uncleared = []
logger.info(event=LogEvent.PruneNeptuneGraphsStart)
all_graph_metadatas = client.get_graph_metadatas(name=config.graph_name)
with logger.bind(neptune_endpoint=str(endpoint)):
for graph_metadata in all_graph_metadatas:
assert graph_metadata.name == config.graph_name
graph_epoch = graph_metadata.end_time
with logger.bind(graph_uri=graph_metadata.uri,
graph_epoch=graph_epoch):
if graph_epoch < oldest_acceptable_graph_epoch:
logger.info(event=LogEvent.PruneNeptuneGraphStart)
try:
client.clear_registered_graph(name=config.graph_name,
uri=graph_metadata.uri)
logger.info(event=LogEvent.PruneNeptuneGraphEnd)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=
f"Error pruning graph {graph_metadata.uri}: {ex}",
)
uncleared.append(graph_metadata.uri)
continue
else:
logger.info(event=LogEvent.PruneNeptuneGraphSkip)
# now find orphaned graphs - these are in neptune but have no metadata
registered_graph_uris = [g_m.uri for g_m in all_graph_metadatas]
all_graph_uris = client.get_graph_uris(name=config.graph_name)
orphaned_graphs = set(all_graph_uris) - set(registered_graph_uris)
if orphaned_graphs:
for orphaned_graph_uri in orphaned_graphs:
with logger.bind(graph_uri=orphaned_graph_uri):
logger.info(event=LogEvent.PruneOrphanedNeptuneGraphStart)
try:
client.clear_graph_data(uri=orphaned_graph_uri)
logger.info(
event=LogEvent.PruneOrphanedNeptuneGraphEnd)
except Exception as ex:
logger.error(
event=LogEvent.PruneNeptuneGraphError,
msg=
f"Error pruning graph {orphaned_graph_uri}: {ex}",
)
uncleared.append(orphaned_graph_uri)
continue
logger.info(event=LogEvent.PruneNeptuneGraphsEnd)
if uncleared:
msg = f"Errors were found pruning {uncleared}."
logger.error(event=LogEvent.PruneNeptuneGraphsError, msg=msg)
raise Exception(msg)