def handler(traversal: GraphTraversal,
resolver_input: ResolverInput) -> Dict:
"""
:param traversal:
:param resolver_input:
:return:
"""
input_dict = resolver_input.arguments.get("input", {})
pagination_info = resolver_input.arguments.get(
"pagination", {
"page": 1,
"per_page": 10
})
page, per_page = pagination_info.get("page"), pagination_info.get(
"per_page")
first, last = get_range(page, per_page)
traversal = traversal_func(traversal, resolver_input)
traversal = filter(traversal, input_dict)
traversal = select(traversal)
response_and_total = traversal.fold().project("data", "total").select("data", "total").\
by(__.unfold().range(first, last).fold()).by(__.unfold().count()).next()
response = [
format(value_map)
for value_map in response_and_total.get("data")
]
return paginate(response, page, per_page,
response_and_total.get("total"))
def loop_data(g,dataset):
print("running ....")
for (index,data) in dataset.iterrows():
# getorcreate node
v1 = g.V().has('sim','name',data["a_number"]).fold().coalesce(__.unfold(),__.addV('sim').property('name',data["a_number"]).property('tac',data["tac"])).next()
v2 = g.V().has('sim','name',data["b_number"]).fold().coalesce(__.unfold(),__.addV('sim').property('name',data["b_number"])).next()
# create egde
g.V(v1).addE('info').to(v2).property('service_type',data["service_type"]).property('b_prefix',data["b_prefix"]).property('start_time',data["start_time"]).property('duration',data["duration"]).iterate()
def __write_edges(self, g: traversal, edges: List[Dict], scan_id: str) -> None:
"""
Writes the edges to the labeled property graph
:param g: The graph traversal source
:param edges: A list of dictionaries for each edge
:return: None
"""
cnt = 0
t = g
for r in edges:
to_id = f'{r["~to"]}_{scan_id}'
from_id = f'{r["~from"]}_{scan_id}'
t = (
t.addE(r["~label"])
.property(T.id, str(r["~id"]))
.from_(
__.V(from_id)
.fold()
.coalesce(
__.unfold(),
__.addV(self.parse_arn(r["~from"])["resource"])
.property(T.id, from_id)
.property("scan_id", scan_id)
.property("arn", r["~from"]),
)
)
.to(
__.V(to_id)
.fold()
.coalesce(
__.unfold(),
__.addV(self.parse_arn(r["~to"])["resource"])
.property(T.id, to_id)
.property("scan_id", scan_id)
.property("arn", r["~to"]),
)
)
)
cnt += 1
if cnt % 100 == 0 or cnt == len(edges):
try:
self.logger.info(
event=LogEvent.NeptunePeriodicWrite,
msg=f"Writing edges {cnt} of {len(edges)}",
)
t.next()
t = g
except Exception as err:
self.logger.error(event=LogEvent.NeptuneLoadError, msg=str(err))
raise NeptuneLoadGraphException(
f"Error loading edge {r} " f"with {str(t.bytecode)}"
) from err
def get_or_create_vertex(self, label_value, id):
return self._do_next(
self.g.V(id).fold().coalesce(
__.unfold(),
__.addV(label_value).property(T.id,
id).property(ID, id).property(
DATE, utils.get_date_now())))
def upsert_vertices_for_label(rows):
conn = self.gremlin_utils.remote_connection()
g = self.gremlin_utils.traversal_source(connection=conn)
t = g
i = 0
for row in rows:
entries = row.asDict()
create_traversal = __.addV(label)
for key, value in entries.items():
key = key.split(':')[0]
if key == '~id':
create_traversal = create_traversal.property(id, value)
elif key == '~label':
pass
else:
create_traversal = create_traversal.property(
key, value)
t = t.V(entries['~id']).fold().coalesce(
__.unfold(), create_traversal)
i += 1
if i == batch_size:
self.retry_query(t)
t = g
i = 0
if i > 0:
self.retry_query(t)
conn.close()
def upsert_vertex(record, vertex_mapping, g):
vertex_label = vertex_mapping['vertex_label']
# Ensure all lookup values are present first
lookup_values = get_lookup_values(record,
vertex_mapping['lookup_properties'])
if lookup_values is None:
return
# Setup traversals
try:
traversal = g.V().hasLabel(vertex_label)
insertion_traversal = __.addV(vertex_label).property(
'type', vertex_label)
for prop_key, lookup_value in lookup_values.items():
traversal = traversal.has(prop_key, lookup_value)
insertion_traversal = insertion_traversal.property(
prop_key, lookup_value)
# Add Vertex insertion partial traversal
for source_field, prop_key in vertex_mapping['other_properties'].items(
):
insertion_traversal = insertion_traversal.property(
prop_key, record[source_field])
traversal.fold().coalesce(__.unfold(), insertion_traversal).next()
except:
print("Vertex error - skipping: {0}({1})".format(
vertex_label, lookup_values))
def add_edges_for_label(rows):
conn = self.gremlin_utils.remote_connection()
g = self.gremlin_utils.traversal_source(connection=conn)
t = g
i = 0
for row in rows:
entries = row.asDict()
create_traversal = __.V(entries['~from']).addE(label).to(
V(entries['~to'])).property(id, entries['~id'])
for key, value in entries.items():
key = key.split(':')[0]
if key not in ['~id', '~from', '~to', '~label']:
create_traversal.property(key, value)
t = t.V(entries['~from']).outE(label).hasId(
entries['~id']).fold().coalesce(__.unfold(),
create_traversal)
i += 1
if i == batch_size:
self.retry_query(t)
t = g
i = 0
if i > 0:
self.retry_query(t)
conn.close()
def __add_author(self, t, author, post_url):
img_src = None
if "img_src" in author.keys():
img_src = author['img_src']
img_height = author['img_height']
img_width = author['img_width']
t = (
t.V(author['name'])
.fold()
.coalesce(
__.unfold(),
__.addV('author')
.property(T.id, author['name'])
.property('name', author['name'])
).as_('p').addE('written_by').from_(__.V(post_url))
)
# Conditionally add the img_src, img_height, and img_width property if they do not exist
if img_src:
t = (
t.sideEffect(
__.select('p').hasNot('img_src')
.property('img_src', img_src)
.property('img_height', img_height)
.property('img_width', img_width)
)
)
return t
def upsert_edge(t, row, **kwargs):
mappings = kwargs['mappings']
label = kwargs['label'] if 'label' in kwargs else mappings.get_label(row)
on_upsert = kwargs.get('on_upsert', None)
#updateSingleCardinalityProperties
#updateAllProperties
#replaceAllProperties
create_traversal = __.addE(label).from_(V(mappings.get_from(row))).to(
V(mappings.get_to(row))).property(id, mappings.get_id(row))
if not on_upsert:
for key, value in row.items():
mapping = mappings.mapping_for(key)
if not mapping.is_token():
create_traversal = create_traversal.property(
mapping.name, mapping.convert(value))
t = t.V(mappings.get_from(row)).outE(label).hasId(
mappings.get_id(row)).fold().coalesce(__.unfold(), create_traversal)
if on_upsert and on_upsert in [
'updateSingleCardinalityProperties', 'updateAllProperties'
]:
for key, value in row.items():
mapping = mappings.mapping_for(key)
if not mapping.is_token():
t = t.property(mapping.name, mapping.convert(value))
return t
def _build_gremlin_edges(g: GraphTraversalSource,
row: pd.Series) -> GraphTraversalSource:
g = (g.V(str(row["~from"])).fold().coalesce(
__.unfold(),
_build_gremlin_vertices(__, {
"~id": row["~from"],
"~label": "Vertex"
})).addE(row["~label"]).to(
__.V(str(row["~to"])).fold().coalesce(
__.unfold(),
_build_gremlin_vertices(__, {
"~id": row["~to"],
"~label": "Vertex"
}))))
g = _build_gremlin_properties(g, row)
return g
def list_users():
graph = Graph()
remote = DriverRemoteConnection('ws://' + os.environ['DB_ENDPOINT'] + ':8182/gremlin', 'g')
g = graph.traversal().withRemote(remote)
print(g.V().hasLabel('user').valueMap(True).by(__.unfold()).next())
return jsonify(user='******'), 200
def _build_gremlin_vertices(g: GraphTraversalSource,
row: Any) -> GraphTraversalSource:
g = g.V(str(row["~id"])).fold().coalesce(
__.unfold(),
__.addV(row["~label"]).property(T.id, str(row["~id"])))
g = _build_gremlin_properties(g, row)
return g
def upsert_node(self, node_id: str, node_label: str,
node_properties: Dict[str, Any]) -> None:
create_traversal = __.addV(node_label).property(T.id, node_id)
node_traversal = self.get_graph().V().has(T.id, node_id). \
fold().coalesce(__.unfold(), create_traversal)
node_traversal = NeptuneSessionClient.update_entity_properties_on_traversal(
node_traversal, node_properties)
node_traversal.next()
def _build_gremlin_insert_vertices(
g: GraphTraversalSource,
row: Any,
use_header_cardinality: bool = False) -> GraphTraversalSource:
g = g.V(str(row["~id"])).fold().coalesce(
__.unfold(),
__.addV(row["~label"]).property(T.id, str(row["~id"])))
g = _set_properties(g, use_header_cardinality, row)
return g
def _build_gremlin_insert_edges(
g: GraphTraversalSource, row: pd.Series,
use_header_cardinality: bool) -> GraphTraversalSource:
g = (g.V(str(row["~from"])).fold().coalesce(
__.unfold(),
_build_gremlin_insert_vertices(__, {
"~id": row["~from"],
"~label": "Vertex"
})).addE(row["~label"]).property(T.id, str(row["~id"])).to(
__.V(str(row["~to"])).fold().coalesce(
__.unfold(),
_build_gremlin_insert_vertices(__, {
"~id": row["~to"],
"~label": "Vertex"
}))))
g = _set_properties(g, use_header_cardinality, row)
return g
def _write_vertex(self, vertex_id: str,
vertex_labels: List[str]) -> Traversal:
logger.debug("Writing vertex %s", vertex_id)
if self.supports_multiple_labels:
vertex_label = "::".join(vertex_labels)
else:
vertex_label = vertex_labels[0]
return self.g.V(vertex_id).fold().coalesce(
__.unfold(),
__.addV(vertex_label).property(T.id, vertex_id))
def test_explain(self) -> None:
proxy = self.get_proxy()
g = proxy.g.V().has(VertexTypes.User.value.label, proxy.key_property_name, 'jack').fold().coalesce(
__.unfold(),
__.addV(VertexTypes.User.value.label).property(Cardinality.single, proxy.key_property_name, 'jack'))
g = g.property(Cardinality.single, 'email', '*****@*****.**')
query = ScriptTranslator.translateT(g)
g.iterate()
# just enough to not explode
proxy._explain(query)
def upsert_edge(self, start_node_id: str, end_node_id: str, edge_id: str,
edge_label: str, edge_properties: Dict[str, Any]) -> None:
create_traversal = __.V().has(T.id, start_node_id).addE(edge_label).to(
__.V().has(T.id, end_node_id)).property(T.id, edge_id)
edge_traversal = self.get_graph().V().has(T.id, start_node_id).outE(edge_label).has(T.id, edge_id). \
fold(). \
coalesce(__.unfold(), create_traversal)
edge_traversal = NeptuneSessionClient.update_entity_properties_on_traversal(
edge_traversal, edge_properties)
edge_traversal.next()
def get_last_checkpoint(client, tablename):
conn = client.remote_connection()
g = client.traversal_source(conn)
checkpoint = (g.V().hasLabel('Checkpoint').has(
'table', tablename).fold().coalesce(
__.unfold(),
__.addV('Checkpoint').property('table', tablename).property(
'value', datetime.datetime(2015, 1, 1, 0,
0))).values('value').next())
conn.close()
return checkpoint
def __add_tag(self, t, tag, post_url):
t = (
t.V(tag)
.fold()
.coalesce(
__.unfold(),
__.addV('tag')
.property(T.id, tag)
.property('tag', tag)
).addE('tagged').from_(__.V(post_url))
)
return t
def add_edges_for_label(rows):
conn = self.remote_connection()
g = self.traversal_source(conn)
for row in rows:
entries = row.asDict()
create_traversal = __.V(row['~from']).addE(label).to(
V(row['~to'])).property(id, row['~id'])
for key, value in entries.iteritems():
key = key.split(':')[0]
if key not in ['~id', '~from', '~to', '~label']:
create_traversal.property(key, value)
g.E(entries['~id']).fold().coalesce(__.unfold(),
create_traversal).next()
conn.close()
def list_user(username):
graph = Graph()
remote = DriverRemoteConnection('ws://' + os.environ['DB_ENDPOINT'] + ':8182/gremlin', 'g')
g = graph.traversal().withRemote(remote)
user = g.V().hasLabel('user').has('username', username).valueMap(False).by(__.unfold())
if not user.hasNext():
content = { 'response': '404: User not found' }
return content, 404
properties = user.next()
return jsonify(user=properties), 200
def __add_entities(self, t, entity, post_url):
t = (
t.V(f'{entity["Text"]}_{entity["Type"]}')
.fold()
.coalesce(
__.unfold(),
__.addV(entity["Type"].lower())
.property(T.id, f'{entity["Text"]}_{entity["Type"]}')
.property("text", entity["Text"])
.property("type", entity["Type"])
).addE('found_in').from_(__.V(post_url))
.property('score', entity['Score'])
)
return t
def upsert_vertex(t, row, **kwargs):
mappings = kwargs['mappings']
label = kwargs['label'] if 'label' in kwargs else mappings.get_label(row)
on_upsert = kwargs.get('on_upsert', None)
#updateSingleCardinalityProperties
#updateAllProperties
#replaceAllProperties
create_traversal = __.addV(label)
updateable_items = []
for key, value in row.items():
mapping = mappings.mapping_for(key)
if mapping.is_id_token():
create_traversal = create_traversal.property(id, value)
elif not mapping.is_token():
if not on_upsert:
create_traversal = create_traversal.property(
mapping.name, mapping.convert(value))
elif on_upsert == 'updateSingleCardinalityProperties':
if mapping.cardinality == 'single':
updateable_items.append((key, value))
else:
create_traversal = create_traversal.property(
get_cardinality(mapping.cardinality), mapping.name,
mapping.convert(value))
elif on_upsert == 'updateAllProperties':
updateable_items.append((key, value))
elif on_upsert == 'replaceAllProperties':
pass
t = t.V(mappings.get_id(row)).fold().coalesce(__.unfold(),
create_traversal)
if updateable_items:
for key, value in updateable_items:
mapping = mappings.mapping_for(key)
t = t.property(get_cardinality(mapping.cardinality), mapping.name,
mapping.convert(value))
return t
def __add_post(self, t, post):
t = (
t.V(post['url'])
.fold()
.coalesce(
__.unfold(),
__.addV('post')
.property(T.id, post['url'])
.property('title', post['title'])
.property('post_date', post['date'])
.property('img_src', post['img_src'])
.property('img_height', post['img_height'])
.property('img_width', post['img_width'])
).as_('post')
)
return t
def upsert_vertices_for_label(rows):
conn = self.remote_connection()
g = self.traversal_source(conn)
for row in rows:
entries = row.asDict()
create_traversal = __.addV(label)
for key, value in entries.iteritems():
key = key.split(':')[0]
if key == '~id':
create_traversal.property(id, value)
elif key == '~label':
pass
else:
create_traversal.property(key, value)
g.V(entries['~id']).fold().coalesce(__.unfold(),
create_traversal).next()
conn.close()
def add_edges_for_label(rows):
try:
conn = self.gremlin_utils.remote_connection()
g = self.gremlin_utils.traversal_source(connection=conn)
for row in rows:
entries = row.asDict()
create_traversal = __.V(row['~from']).addE(label).to(
V(row['~to'])).property(id, row['~id'])
for key, value in entries.items():
key = key.split(':')[0]
if key not in ['~id', '~from', '~to', '~label']:
create_traversal.property(key, value)
g.E(entries['~id']).fold().coalesce(
__.unfold(), create_traversal).next()
conn.close()
except GremlinServerError as err:
print("Neptune error: {0}".format(err))
except:
print("Unexpected error:", sys.exc_info()[0])
def upsert_edge(record, edge_mapping, g):
edge_label = edge_mapping['edge_label']
# Simple logic, requiring that Vertices must exist before edge can be added.
# Ensure all lookup values are present first
out_lookup_values = get_lookup_values(
record, edge_mapping['out_vertex']['lookup_properties'])
in_lookup_values = get_lookup_values(
record, edge_mapping['in_vertex']['lookup_properties'])
if out_lookup_values is None or in_lookup_values is None:
return
try:
traversal = g.V().hasLabel(edge_mapping['out_vertex']['vertex_label'])
insertion_traversal = __.V().hasLabel(
edge_mapping['out_vertex']['vertex_label'])
for prop_key, lookup_value in out_lookup_values.items():
traversal = traversal.has(prop_key, lookup_value)
insertion_traversal = insertion_traversal.has(
prop_key, lookup_value)
traversal = traversal.as_('out').V().hasLabel(
edge_mapping['in_vertex']['vertex_label'])
insertion_traversal = insertion_traversal.as_('out2').V().hasLabel(
edge_mapping['in_vertex']['vertex_label'])
for prop_key, lookup_value in in_lookup_values.items():
traversal = traversal.has(prop_key, lookup_value)
insertion_traversal = insertion_traversal.has(
prop_key, lookup_value)
insertion_traversal = insertion_traversal.addE(edge_label).from_(
'out2')
traversal = traversal.as_('in').inE(edge_label).as_('e').outV().where(
P.eq('out')).fold().coalesce(__.unfold(),
insertion_traversal).next()
except:
print("Edge error - skipping: {0}({1}) --{2}-> {3}({4})".format(
edge_mapping['out_vertex']['vertex_label'], out_lookup_values,
edge_label, edge_mapping['in_vertex']['vertex_label'],
in_lookup_values))
def __write_vertices(self, g: traversal, vertices: List[Dict], scan_id: str) -> None:
"""
Writes the vertices to the labeled property graph
:param g: The graph traversal source
:param vertices: A list of dictionaries for each vertex
:return: None
"""
cnt = 0
t = g
for r in vertices:
vertex_id = f'{r["~id"]}_{scan_id}'
t = (
t.V(vertex_id)
.fold()
.coalesce(
__.unfold(),
__.addV(self.parse_arn(r["~label"])["resource"]).property(T.id, vertex_id),
)
)
for k in r.keys():
# Need to handle numbers that are bigger than a Long in Java, for now we stringify it
if isinstance(r[k], int) and (
r[k] > 9223372036854775807 or r[k] < -9223372036854775807
):
r[k] = str(r[k])
if k not in ["~id", "~label"]:
t = t.property(k, r[k])
cnt += 1
if cnt % 100 == 0 or cnt == len(vertices):
try:
self.logger.info(
event=LogEvent.NeptunePeriodicWrite,
msg=f"Writing vertices {cnt} of {len(vertices)}",
)
t.next()
t = g
except Exception as err:
print(str(err))
raise NeptuneLoadGraphException(
f"Error loading vertex {r} " f"with {str(t.bytecode)}"
) from err
def upsert_vertices_for_label(rows):
try:
conn = self.gremlin_utils.remote_connection()
g = self.gremlin_utils.traversal_source(connection=conn)
for row in rows:
entries = row.asDict()
create_traversal = __.addV(label)
for key, value in entries.items():
key = key.split(':')[0]
if key == '~id':
create_traversal.property(id, value)
elif key == '~label':
pass
else:
create_traversal.property(key, value)
g.V(entries['~id']).fold().coalesce(
__.unfold(), create_traversal).next()
conn.close()
except GremlinServerError as err:
print("Neptune error: {0}".format(err))
except:
print("Unexpected error:", sys.exc_info()[0])
