def __init__(self, neptune_endpoint, elasticache_endpoint):
GremlinUtils.init_statics(globals())
gremlin_utils = GremlinUtils(
Endpoints(neptune_endpoint=neptune_endpoint))
self.vertext_metrics = VertexMetrics(elasticache_endpoint)
self.neptune_connection = gremlin_utils.remote_connection()
self.g = gremlin_utils.traversal_source(
connection=self.neptune_connection)
def handle_records(self, stream_log):
params = json.loads(os.environ['AdditionalParams'])
neptune_endpoint = params['neptune_cluster_endpoint']
neptune_port = params['neptune_port']
GremlinUtils.init_statics(globals())
endpoints = Endpoints(neptune_endpoint=neptune_endpoint,
neptune_port=neptune_port)
gremlin_utils = GremlinUtils(endpoints)
conn = gremlin_utils.remote_connection()
g = gremlin_utils.traversal_source(connection=conn)
records = stream_log[RECORDS_STR]
last_op_num = None
last_commit_num = None
count = 0
try:
for record in records:
# Process record
op = record[OPERATION_STR]
data = record[DATA_STR]
type = data['type']
id = data['id']
if op == ADD_OPERATION:
if type == 'vl':
logger.info(g.V(id).valueMap(True).toList())
if type == 'e':
logger.info(g.E(id).valueMap(True).toList())
# Update local checkpoint info
last_op_num = record[EVENT_ID_STR][OP_NUM_STR]
last_commit_num = record[EVENT_ID_STR][COMMIT_NUM_STR]
count += 1
except Exception as e:
logger.error('Error occurred - {}'.format(str(e)))
raise e
finally:
try:
conn.close()
yield HandlerResponse(last_op_num, last_commit_num, count)
except Exception as e:
logger.error('Error occurred - {}'.format(str(e)))
raise e
finally:
conn.close()
def __init__(self, endpoints):
self.gremlin_utils = GremlinUtils(endpoints)
GremlinUtils.init_statics(globals())
import traceback
from gremlin_python import statics
from gremlin_python.structure.graph import Graph
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.strategies import *
from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
from gremlin_python.driver.protocol import GremlinServerError
from gremlin_python.process.traversal import *
from neptune_python_utils.gremlin_utils import GremlinUtils
from neptune_python_utils.endpoints import Endpoints
from neptune_python_utils.mappings import Mappings
logging.getLogger('backoff').addHandler(logging.StreamHandler())
logger = logging.getLogger()
GremlinUtils.init_statics(globals())
def get_cardinality(s):
return Cardinality.single if s == 'single' else Cardinality.set_
def add_vertex(t, row, **kwargs):
mappings = kwargs['mappings']
label = kwargs['label'] if 'label' in kwargs else mappings.get_label(row)
t = t.addV(label)
for key, value in row.items():
mapping = mappings.mapping_for(key)
def __init__(self):
GremlinUtils.init_statics(globals())
gremlin_utils = GremlinUtils()
self.neptune_connection = gremlin_utils.remote_connection()
self.g = gremlin_utils.traversal_source(
connection=self.neptune_connection)