def handler(traversal: GraphTraversal, resolver_input: ResolverInput) -> Dict: """ :param traversal: :param resolver_input: :return: """ input_dict = resolver_input.arguments.get("input", {}) pagination_info = resolver_input.arguments.get( "pagination", { "page": 1, "per_page": 10 }) page, per_page = pagination_info.get("page"), pagination_info.get( "per_page") first, last = get_range(page, per_page) traversal = traversal_func(traversal, resolver_input) traversal = filter(traversal, input_dict) traversal = select(traversal) response_and_total = traversal.fold().project("data", "total").select("data", "total").\ by(__.unfold().range(first, last).fold()).by(__.unfold().count()).next() response = [ format(value_map) for value_map in response_and_total.get("data") ] return paginate(response, page, per_page, response_and_total.get("total"))
def loop_data(g,dataset): print("running ....") for (index,data) in dataset.iterrows(): # getorcreate node v1 = g.V().has('sim','name',data["a_number"]).fold().coalesce(__.unfold(),__.addV('sim').property('name',data["a_number"]).property('tac',data["tac"])).next() v2 = g.V().has('sim','name',data["b_number"]).fold().coalesce(__.unfold(),__.addV('sim').property('name',data["b_number"])).next() # create egde g.V(v1).addE('info').to(v2).property('service_type',data["service_type"]).property('b_prefix',data["b_prefix"]).property('start_time',data["start_time"]).property('duration',data["duration"]).iterate()
def __write_edges(self, g: traversal, edges: List[Dict], scan_id: str) -> None: """ Writes the edges to the labeled property graph :param g: The graph traversal source :param edges: A list of dictionaries for each edge :return: None """ cnt = 0 t = g for r in edges: to_id = f'{r["~to"]}_{scan_id}' from_id = f'{r["~from"]}_{scan_id}' t = ( t.addE(r["~label"]) .property(T.id, str(r["~id"])) .from_( __.V(from_id) .fold() .coalesce( __.unfold(), __.addV(self.parse_arn(r["~from"])["resource"]) .property(T.id, from_id) .property("scan_id", scan_id) .property("arn", r["~from"]), ) ) .to( __.V(to_id) .fold() .coalesce( __.unfold(), __.addV(self.parse_arn(r["~to"])["resource"]) .property(T.id, to_id) .property("scan_id", scan_id) .property("arn", r["~to"]), ) ) ) cnt += 1 if cnt % 100 == 0 or cnt == len(edges): try: self.logger.info( event=LogEvent.NeptunePeriodicWrite, msg=f"Writing edges {cnt} of {len(edges)}", ) t.next() t = g except Exception as err: self.logger.error(event=LogEvent.NeptuneLoadError, msg=str(err)) raise NeptuneLoadGraphException( f"Error loading edge {r} " f"with {str(t.bytecode)}" ) from err
def get_or_create_vertex(self, label_value, id): return self._do_next( self.g.V(id).fold().coalesce( __.unfold(), __.addV(label_value).property(T.id, id).property(ID, id).property( DATE, utils.get_date_now())))
def upsert_vertices_for_label(rows): conn = self.gremlin_utils.remote_connection() g = self.gremlin_utils.traversal_source(connection=conn) t = g i = 0 for row in rows: entries = row.asDict() create_traversal = __.addV(label) for key, value in entries.items(): key = key.split(':')[0] if key == '~id': create_traversal = create_traversal.property(id, value) elif key == '~label': pass else: create_traversal = create_traversal.property( key, value) t = t.V(entries['~id']).fold().coalesce( __.unfold(), create_traversal) i += 1 if i == batch_size: self.retry_query(t) t = g i = 0 if i > 0: self.retry_query(t) conn.close()
def upsert_vertex(record, vertex_mapping, g): vertex_label = vertex_mapping['vertex_label'] # Ensure all lookup values are present first lookup_values = get_lookup_values(record, vertex_mapping['lookup_properties']) if lookup_values is None: return # Setup traversals try: traversal = g.V().hasLabel(vertex_label) insertion_traversal = __.addV(vertex_label).property( 'type', vertex_label) for prop_key, lookup_value in lookup_values.items(): traversal = traversal.has(prop_key, lookup_value) insertion_traversal = insertion_traversal.property( prop_key, lookup_value) # Add Vertex insertion partial traversal for source_field, prop_key in vertex_mapping['other_properties'].items( ): insertion_traversal = insertion_traversal.property( prop_key, record[source_field]) traversal.fold().coalesce(__.unfold(), insertion_traversal).next() except: print("Vertex error - skipping: {0}({1})".format( vertex_label, lookup_values))
def add_edges_for_label(rows): conn = self.gremlin_utils.remote_connection() g = self.gremlin_utils.traversal_source(connection=conn) t = g i = 0 for row in rows: entries = row.asDict() create_traversal = __.V(entries['~from']).addE(label).to( V(entries['~to'])).property(id, entries['~id']) for key, value in entries.items(): key = key.split(':')[0] if key not in ['~id', '~from', '~to', '~label']: create_traversal.property(key, value) t = t.V(entries['~from']).outE(label).hasId( entries['~id']).fold().coalesce(__.unfold(), create_traversal) i += 1 if i == batch_size: self.retry_query(t) t = g i = 0 if i > 0: self.retry_query(t) conn.close()
def __add_author(self, t, author, post_url): img_src = None if "img_src" in author.keys(): img_src = author['img_src'] img_height = author['img_height'] img_width = author['img_width'] t = ( t.V(author['name']) .fold() .coalesce( __.unfold(), __.addV('author') .property(T.id, author['name']) .property('name', author['name']) ).as_('p').addE('written_by').from_(__.V(post_url)) ) # Conditionally add the img_src, img_height, and img_width property if they do not exist if img_src: t = ( t.sideEffect( __.select('p').hasNot('img_src') .property('img_src', img_src) .property('img_height', img_height) .property('img_width', img_width) ) ) return t
def upsert_edge(t, row, **kwargs): mappings = kwargs['mappings'] label = kwargs['label'] if 'label' in kwargs else mappings.get_label(row) on_upsert = kwargs.get('on_upsert', None) #updateSingleCardinalityProperties #updateAllProperties #replaceAllProperties create_traversal = __.addE(label).from_(V(mappings.get_from(row))).to( V(mappings.get_to(row))).property(id, mappings.get_id(row)) if not on_upsert: for key, value in row.items(): mapping = mappings.mapping_for(key) if not mapping.is_token(): create_traversal = create_traversal.property( mapping.name, mapping.convert(value)) t = t.V(mappings.get_from(row)).outE(label).hasId( mappings.get_id(row)).fold().coalesce(__.unfold(), create_traversal) if on_upsert and on_upsert in [ 'updateSingleCardinalityProperties', 'updateAllProperties' ]: for key, value in row.items(): mapping = mappings.mapping_for(key) if not mapping.is_token(): t = t.property(mapping.name, mapping.convert(value)) return t
def _build_gremlin_edges(g: GraphTraversalSource, row: pd.Series) -> GraphTraversalSource: g = (g.V(str(row["~from"])).fold().coalesce( __.unfold(), _build_gremlin_vertices(__, { "~id": row["~from"], "~label": "Vertex" })).addE(row["~label"]).to( __.V(str(row["~to"])).fold().coalesce( __.unfold(), _build_gremlin_vertices(__, { "~id": row["~to"], "~label": "Vertex" })))) g = _build_gremlin_properties(g, row) return g
def list_users(): graph = Graph() remote = DriverRemoteConnection('ws://' + os.environ['DB_ENDPOINT'] + ':8182/gremlin', 'g') g = graph.traversal().withRemote(remote) print(g.V().hasLabel('user').valueMap(True).by(__.unfold()).next()) return jsonify(user='******'), 200
def _build_gremlin_vertices(g: GraphTraversalSource, row: Any) -> GraphTraversalSource: g = g.V(str(row["~id"])).fold().coalesce( __.unfold(), __.addV(row["~label"]).property(T.id, str(row["~id"]))) g = _build_gremlin_properties(g, row) return g
def upsert_node(self, node_id: str, node_label: str, node_properties: Dict[str, Any]) -> None: create_traversal = __.addV(node_label).property(T.id, node_id) node_traversal = self.get_graph().V().has(T.id, node_id). \ fold().coalesce(__.unfold(), create_traversal) node_traversal = NeptuneSessionClient.update_entity_properties_on_traversal( node_traversal, node_properties) node_traversal.next()
def _build_gremlin_insert_vertices( g: GraphTraversalSource, row: Any, use_header_cardinality: bool = False) -> GraphTraversalSource: g = g.V(str(row["~id"])).fold().coalesce( __.unfold(), __.addV(row["~label"]).property(T.id, str(row["~id"]))) g = _set_properties(g, use_header_cardinality, row) return g
def _build_gremlin_insert_edges( g: GraphTraversalSource, row: pd.Series, use_header_cardinality: bool) -> GraphTraversalSource: g = (g.V(str(row["~from"])).fold().coalesce( __.unfold(), _build_gremlin_insert_vertices(__, { "~id": row["~from"], "~label": "Vertex" })).addE(row["~label"]).property(T.id, str(row["~id"])).to( __.V(str(row["~to"])).fold().coalesce( __.unfold(), _build_gremlin_insert_vertices(__, { "~id": row["~to"], "~label": "Vertex" })))) g = _set_properties(g, use_header_cardinality, row) return g
def _write_vertex(self, vertex_id: str, vertex_labels: List[str]) -> Traversal: logger.debug("Writing vertex %s", vertex_id) if self.supports_multiple_labels: vertex_label = "::".join(vertex_labels) else: vertex_label = vertex_labels[0] return self.g.V(vertex_id).fold().coalesce( __.unfold(), __.addV(vertex_label).property(T.id, vertex_id))
def test_explain(self) -> None: proxy = self.get_proxy() g = proxy.g.V().has(VertexTypes.User.value.label, proxy.key_property_name, 'jack').fold().coalesce( __.unfold(), __.addV(VertexTypes.User.value.label).property(Cardinality.single, proxy.key_property_name, 'jack')) g = g.property(Cardinality.single, 'email', '*****@*****.**') query = ScriptTranslator.translateT(g) g.iterate() # just enough to not explode proxy._explain(query)
def upsert_edge(self, start_node_id: str, end_node_id: str, edge_id: str, edge_label: str, edge_properties: Dict[str, Any]) -> None: create_traversal = __.V().has(T.id, start_node_id).addE(edge_label).to( __.V().has(T.id, end_node_id)).property(T.id, edge_id) edge_traversal = self.get_graph().V().has(T.id, start_node_id).outE(edge_label).has(T.id, edge_id). \ fold(). \ coalesce(__.unfold(), create_traversal) edge_traversal = NeptuneSessionClient.update_entity_properties_on_traversal( edge_traversal, edge_properties) edge_traversal.next()
def get_last_checkpoint(client, tablename): conn = client.remote_connection() g = client.traversal_source(conn) checkpoint = (g.V().hasLabel('Checkpoint').has( 'table', tablename).fold().coalesce( __.unfold(), __.addV('Checkpoint').property('table', tablename).property( 'value', datetime.datetime(2015, 1, 1, 0, 0))).values('value').next()) conn.close() return checkpoint
def __add_tag(self, t, tag, post_url): t = ( t.V(tag) .fold() .coalesce( __.unfold(), __.addV('tag') .property(T.id, tag) .property('tag', tag) ).addE('tagged').from_(__.V(post_url)) ) return t
def add_edges_for_label(rows): conn = self.remote_connection() g = self.traversal_source(conn) for row in rows: entries = row.asDict() create_traversal = __.V(row['~from']).addE(label).to( V(row['~to'])).property(id, row['~id']) for key, value in entries.iteritems(): key = key.split(':')[0] if key not in ['~id', '~from', '~to', '~label']: create_traversal.property(key, value) g.E(entries['~id']).fold().coalesce(__.unfold(), create_traversal).next() conn.close()
def list_user(username): graph = Graph() remote = DriverRemoteConnection('ws://' + os.environ['DB_ENDPOINT'] + ':8182/gremlin', 'g') g = graph.traversal().withRemote(remote) user = g.V().hasLabel('user').has('username', username).valueMap(False).by(__.unfold()) if not user.hasNext(): content = { 'response': '404: User not found' } return content, 404 properties = user.next() return jsonify(user=properties), 200
def __add_entities(self, t, entity, post_url): t = ( t.V(f'{entity["Text"]}_{entity["Type"]}') .fold() .coalesce( __.unfold(), __.addV(entity["Type"].lower()) .property(T.id, f'{entity["Text"]}_{entity["Type"]}') .property("text", entity["Text"]) .property("type", entity["Type"]) ).addE('found_in').from_(__.V(post_url)) .property('score', entity['Score']) ) return t
def upsert_vertex(t, row, **kwargs): mappings = kwargs['mappings'] label = kwargs['label'] if 'label' in kwargs else mappings.get_label(row) on_upsert = kwargs.get('on_upsert', None) #updateSingleCardinalityProperties #updateAllProperties #replaceAllProperties create_traversal = __.addV(label) updateable_items = [] for key, value in row.items(): mapping = mappings.mapping_for(key) if mapping.is_id_token(): create_traversal = create_traversal.property(id, value) elif not mapping.is_token(): if not on_upsert: create_traversal = create_traversal.property( mapping.name, mapping.convert(value)) elif on_upsert == 'updateSingleCardinalityProperties': if mapping.cardinality == 'single': updateable_items.append((key, value)) else: create_traversal = create_traversal.property( get_cardinality(mapping.cardinality), mapping.name, mapping.convert(value)) elif on_upsert == 'updateAllProperties': updateable_items.append((key, value)) elif on_upsert == 'replaceAllProperties': pass t = t.V(mappings.get_id(row)).fold().coalesce(__.unfold(), create_traversal) if updateable_items: for key, value in updateable_items: mapping = mappings.mapping_for(key) t = t.property(get_cardinality(mapping.cardinality), mapping.name, mapping.convert(value)) return t
def __add_post(self, t, post): t = ( t.V(post['url']) .fold() .coalesce( __.unfold(), __.addV('post') .property(T.id, post['url']) .property('title', post['title']) .property('post_date', post['date']) .property('img_src', post['img_src']) .property('img_height', post['img_height']) .property('img_width', post['img_width']) ).as_('post') ) return t
def upsert_vertices_for_label(rows): conn = self.remote_connection() g = self.traversal_source(conn) for row in rows: entries = row.asDict() create_traversal = __.addV(label) for key, value in entries.iteritems(): key = key.split(':')[0] if key == '~id': create_traversal.property(id, value) elif key == '~label': pass else: create_traversal.property(key, value) g.V(entries['~id']).fold().coalesce(__.unfold(), create_traversal).next() conn.close()
def add_edges_for_label(rows): try: conn = self.gremlin_utils.remote_connection() g = self.gremlin_utils.traversal_source(connection=conn) for row in rows: entries = row.asDict() create_traversal = __.V(row['~from']).addE(label).to( V(row['~to'])).property(id, row['~id']) for key, value in entries.items(): key = key.split(':')[0] if key not in ['~id', '~from', '~to', '~label']: create_traversal.property(key, value) g.E(entries['~id']).fold().coalesce( __.unfold(), create_traversal).next() conn.close() except GremlinServerError as err: print("Neptune error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0])
def upsert_edge(record, edge_mapping, g): edge_label = edge_mapping['edge_label'] # Simple logic, requiring that Vertices must exist before edge can be added. # Ensure all lookup values are present first out_lookup_values = get_lookup_values( record, edge_mapping['out_vertex']['lookup_properties']) in_lookup_values = get_lookup_values( record, edge_mapping['in_vertex']['lookup_properties']) if out_lookup_values is None or in_lookup_values is None: return try: traversal = g.V().hasLabel(edge_mapping['out_vertex']['vertex_label']) insertion_traversal = __.V().hasLabel( edge_mapping['out_vertex']['vertex_label']) for prop_key, lookup_value in out_lookup_values.items(): traversal = traversal.has(prop_key, lookup_value) insertion_traversal = insertion_traversal.has( prop_key, lookup_value) traversal = traversal.as_('out').V().hasLabel( edge_mapping['in_vertex']['vertex_label']) insertion_traversal = insertion_traversal.as_('out2').V().hasLabel( edge_mapping['in_vertex']['vertex_label']) for prop_key, lookup_value in in_lookup_values.items(): traversal = traversal.has(prop_key, lookup_value) insertion_traversal = insertion_traversal.has( prop_key, lookup_value) insertion_traversal = insertion_traversal.addE(edge_label).from_( 'out2') traversal = traversal.as_('in').inE(edge_label).as_('e').outV().where( P.eq('out')).fold().coalesce(__.unfold(), insertion_traversal).next() except: print("Edge error - skipping: {0}({1}) --{2}-> {3}({4})".format( edge_mapping['out_vertex']['vertex_label'], out_lookup_values, edge_label, edge_mapping['in_vertex']['vertex_label'], in_lookup_values))
def __write_vertices(self, g: traversal, vertices: List[Dict], scan_id: str) -> None: """ Writes the vertices to the labeled property graph :param g: The graph traversal source :param vertices: A list of dictionaries for each vertex :return: None """ cnt = 0 t = g for r in vertices: vertex_id = f'{r["~id"]}_{scan_id}' t = ( t.V(vertex_id) .fold() .coalesce( __.unfold(), __.addV(self.parse_arn(r["~label"])["resource"]).property(T.id, vertex_id), ) ) for k in r.keys(): # Need to handle numbers that are bigger than a Long in Java, for now we stringify it if isinstance(r[k], int) and ( r[k] > 9223372036854775807 or r[k] < -9223372036854775807 ): r[k] = str(r[k]) if k not in ["~id", "~label"]: t = t.property(k, r[k]) cnt += 1 if cnt % 100 == 0 or cnt == len(vertices): try: self.logger.info( event=LogEvent.NeptunePeriodicWrite, msg=f"Writing vertices {cnt} of {len(vertices)}", ) t.next() t = g except Exception as err: print(str(err)) raise NeptuneLoadGraphException( f"Error loading vertex {r} " f"with {str(t.bytecode)}" ) from err
def upsert_vertices_for_label(rows): try: conn = self.gremlin_utils.remote_connection() g = self.gremlin_utils.traversal_source(connection=conn) for row in rows: entries = row.asDict() create_traversal = __.addV(label) for key, value in entries.items(): key = key.split(':')[0] if key == '~id': create_traversal.property(id, value) elif key == '~label': pass else: create_traversal.property(key, value) g.V(entries['~id']).fold().coalesce( __.unfold(), create_traversal).next() conn.close() except GremlinServerError as err: print("Neptune error: {0}".format(err)) except: print("Unexpected error:", sys.exc_info()[0])