def __init__(self, neptune_endpoint, elasticache_endpoint):
GremlinUtils.init_statics(globals())
gremlin_utils = GremlinUtils(
Endpoints(neptune_endpoint=neptune_endpoint))
self.vertext_metrics = VertexMetrics(elasticache_endpoint)
self.neptune_connection = gremlin_utils.remote_connection()
self.g = gremlin_utils.traversal_source(
connection=self.neptune_connection)
def remoteConnection(self,
neptune_endpoint=None,
neptune_port=None,
show_endpoint=True):
connection = GremlinUtils(Endpoints(
neptune_endpoint, neptune_port)).remote_connection(show_endpoint)
self.connections.append(connection)
return connection
def handle_records(self, stream_log):
params = json.loads(os.environ['AdditionalParams'])
neptune_endpoint = params['neptune_cluster_endpoint']
neptune_port = params['neptune_port']
GremlinUtils.init_statics(globals())
endpoints = Endpoints(neptune_endpoint=neptune_endpoint,
neptune_port=neptune_port)
gremlin_utils = GremlinUtils(endpoints)
conn = gremlin_utils.remote_connection()
g = gremlin_utils.traversal_source(connection=conn)
records = stream_log[RECORDS_STR]
last_op_num = None
last_commit_num = None
count = 0
try:
for record in records:
# Process record
op = record[OPERATION_STR]
data = record[DATA_STR]
type = data['type']
id = data['id']
if op == ADD_OPERATION:
if type == 'vl':
logger.info(g.V(id).valueMap(True).toList())
if type == 'e':
logger.info(g.E(id).valueMap(True).toList())
# Update local checkpoint info
last_op_num = record[EVENT_ID_STR][OP_NUM_STR]
last_commit_num = record[EVENT_ID_STR][COMMIT_NUM_STR]
count += 1
except Exception as e:
logger.error('Error occurred - {}'.format(str(e)))
raise e
finally:
try:
conn.close()
yield HandlerResponse(last_op_num, last_commit_num, count)
except Exception as e:
logger.error('Error occurred - {}'.format(str(e)))
raise e
finally:
conn.close()
def graphTraversal(self,
neptune_endpoint=None,
neptune_port=None,
show_endpoint=True,
connection=None):
if connection is None:
connection = self.remoteConnection(neptune_endpoint, neptune_port,
show_endpoint)
self.connections.append(connection)
return GremlinUtils(Endpoints(neptune_endpoint,
neptune_port)).traversal_source(
show_endpoint, connection)
def __init__(self,
endpoints,
job_name=None,
to_dict=lambda x: x,
pool_size=1,
**kwargs):
self.gremlin_utils = GremlinUtils(endpoints)
self.conn = None
self.g = None
self.region = endpoints.region
self.job_name = job_name
self.to_dict = to_dict
self.pool_size = pool_size
self.kwargs = kwargs
def __init__(self, endpoints):
self.gremlin_utils = GremlinUtils(endpoints)
GremlinUtils.init_statics(globals())
sc = SparkContext()
glueContext = GlueContext(sc)
job = Job(glueContext)
job.init(args['JOB_NAME'], args)
database = args['DATABASE_NAME']
order_table = '{}salesdb_sales_order'.format(args['TABLE_PREFIX'])
order_detail_table = '{}salesdb_sales_order_detail'.format(
args['TABLE_PREFIX'])
gremlin_endpoints = GlueNeptuneConnectionInfo(
args['AWS_REGION'], args['CONNECT_TO_NEPTUNE_ROLE_ARN']).neptune_endpoints(
args['NEPTUNE_CONNECTION_NAME'])
gremlin_client = GlueGremlinClient(gremlin_endpoints)
gremlin_utils = GremlinUtils(gremlin_endpoints)
def get_last_checkpoint(client, tablename):
conn = client.remote_connection()
g = client.traversal_source(connection=conn)
checkpoint = (g.V().hasLabel('Checkpoint').has(
'table', tablename).fold().coalesce(
__.unfold(),
__.addV('Checkpoint').property('table', tablename).property(
'value', datetime.datetime(2015, 1, 1, 0,
0))).values('value').next())
conn.close()
return checkpoint
def __init__(self):
GremlinUtils.init_statics(globals())
gremlin_utils = GremlinUtils()
self.neptune_connection = gremlin_utils.remote_connection()
self.g = gremlin_utils.traversal_source(
connection=self.neptune_connection)