def processItem(item): neo4jGraph, topologicGraph, labelKey, relationshipKey, bidirectional, deleteAll, tolerance, run = item if not (run): return None import time gmt = time.gmtime() timestamp = str(gmt.tm_zone) + "_" + str(gmt.tm_year) + "_" + str( gmt.tm_mon) + "_" + str(gmt.tm_wday) + "_" + str( gmt.tm_hour) + "_" + str(gmt.tm_min) + "_" + str(gmt.tm_sec) vertices = [] _ = topologicGraph.Vertices(vertices) edges = [] _ = topologicGraph.Edges(edges) notUsed = [] tx = neo4jGraph.begin() nodes = [] for i in range(len(vertices)): vDict = vertices[i].GetDictionary() keys, values = getKeysAndValues(vDict) keys.append("x") keys.append("y") keys.append("z") keys.append("timestamp") keys.append("location") values.append(vertices[i].X()) values.append(vertices[i].Y()) values.append(vertices[i].Z()) values.append(timestamp) values.append( sp.CartesianPoint( [vertices[i].X(), vertices[i].Y(), vertices[i].Z()])) zip_iterator = zip(keys, values) pydict = dict(zip_iterator) if (labelKey == 'None') or (not (labelKey)): nodeName = "TopologicGraphVertex" else: nodeName = str(getValueAtKey(vDict, labelKey)) n = py2neo.Node(nodeName, **pydict) tx.create(n) nodes.append(n) for i in range(len(edges)): e = edges[i] sv = e.StartVertex() ev = e.EndVertex() sn = nodes[vertexIndex(sv, vertices, tolerance)] en = nodes[vertexIndex(ev, vertices, tolerance)] ed = e.GetDictionary() relationshipType = getValueAtKey(ed, relationshipKey) if not (relationshipType): relationshipType = "Connected To" snen = py2neo.Relationship(sn, relationshipType, en) tx.create(snen) if bidirectional: snen = py2neo.Relationship(en, relationshipType, sn) tx.create(snen) if deleteAll: neo4jGraph.delete_all() neo4jGraph.commit(tx) return neo4jGraph
def create_followers(node): """ 为参数结点在数据库中生成follower :param node: label为developer的结点 :return: """ info = {} try: info = requests.get(node["followers_url"], headers=headers).json() except KeyError: print("Node error") exit(-1) if info: followers = [member["url"] for member in info] for follower in followers: follower_info = requests.get(follower, headers=headers).json() properties, relationships = get_developer_info(follower_info) temp = properties.copy() temp.update(relationships) follower_node = py2neo.Node("DEVELOPER", **temp) rel1 = py2neo.Relationship(node, "IS_FOLLOWED_BY", follower_node) rel2 = py2neo.Relationship(follower_node, "FOLLOWS", node) tx.merge(follower_node, "DEVELOPER", "id") tx.create(rel1) tx.create(rel2)
def authority_portrait(authority): query = authority.node.properties["person_name_absolute"] print("Portrait {} : {}".format(authority.node.properties["id"], query), flush=True) portraits = Portraits(query) for portrait in portraits: portrait_node = create_entity(portrait) graph.create_unique( py2neo.Relationship(authority.node, "subject_of", portrait_node)) graph.create_unique( py2neo.Relationship(authority.node, "portrait_of", portrait_node))
def processItem(item): import time gmt = time.gmtime() timestamp = str(gmt.tm_zone)+"_"+str(gmt.tm_year)+"_"+str(gmt.tm_mon)+"_"+str(gmt.tm_wday)+"_"+str(gmt.tm_hour)+"_"+str(gmt.tm_min)+"_"+str(gmt.tm_sec) neo4jGraph, topologicGraph, categoryKey, tolerance = item vertices = [] _ = topologicGraph.Vertices(vertices) edges = [] _ = topologicGraph.Edges(edges) notUsed = [] tx = neo4jGraph.begin() nodes = [] for i in range(len(vertices)): vDict = vertices[i].GetDictionary() keys, values = getKeysAndValues(vDict) keys.append("x") keys.append("y") keys.append("z") keys.append("timestamp") keys.append("location") values.append(vertices[i].X()) values.append(vertices[i].Y()) values.append(vertices[i].Z()) values.append(timestamp) values.append(sp.CartesianPoint([vertices[i].X(),vertices[i].Y(),vertices[i].Z()])) zip_iterator = zip(keys, values) pydict = dict(zip_iterator) if categoryKey == 'None': nodeName = "TopologicGraphVertex" else: nodeName = str(values[keys.index(categoryKey)]) n = py2neo.Node(nodeName, **pydict) neo4jGraph.cypher.execute("CREATE INDEX FOR (n:%s) on (n.name)" % n.nodelabel) #try: #neo4jGraph.cypher.execute("CREATE INDEX FOR (n:%s) on (n.name)" % #n.nodelabel) #except: #pass tx.create(n) nodes.append(n) for i in range(len(edges)): e = edges[i] sv = e.StartVertex() ev = e.EndVertex() sn = nodes[vertexIndex(sv, vertices, tolerance)] en = nodes[vertexIndex(ev, vertices, tolerance)] snen = py2neo.Relationship(sn, "CONNECTEDTO", en) tx.create(snen) snen = py2neo.Relationship(en, "CONNECTEDTO", sn) tx.create(snen) neo4jGraph.commit(tx) return neo4jGraph
def create_authentication_failed(from_user_id: str, unverified_tweet_id: str) -> bool: """ to create a relationship called "UNVERIFIED" between a legit User to an impostor tweet from their account. Args: from_user_id (string), person who tweeted, their user id unverified_tweet_id (string), the unauthenticated tweet id, from that user id Returns: boolean, representing whether the relationship got inserted or not. """ matcher = py2neo.NodeMatcher(graph) from_node = matcher.match(USER_NODE_LABEL, user_id=from_user_id).first() to_node = matcher.match(TWEET_NODE_LABEL, tweet_id=unverified_tweet_id).first() failed_auth = py2neo.Relationship( from_node, UNVERIFIED_RELATIONSHIP, to_node, ) graph.create(failed_auth) return graph.exists(failed_auth)
def create_relation(self, source, dest, properties={}, type_=BaseGraphDB.DEFAULT_RELATION): log.log.debug( 'Creating %s from %s to %s with properties %s', type_.name, source.properties['name'], dest.properties['name'], properties ) s = self.get( source.properties['name'], collection=source.collection, db_convert=False ) d = self.get( dest.properties['name'], collection=dest.collection, db_convert=False ) r = py2neo.Relationship(s, type_.name, d, **properties) self._r.create(r) return r
def addRelation(self, fromNode, toNode, relation): """Adds a new relationship between nodes with label.""" try: nodeIDs = self.findNode(fromNode) # add to node add_label(label) if len(nodeIDs) > 1: # print("we got more source nodes with that name") # return raise MultipleNodeFoundError( "Multiple source nodes with this name") if len(nodeIDs) == 1: # one node exists fromNodeID = nodeIDs[0] else: # no node exists, create new node fromNodeID = self.createNewNode(fromNode) nodeIDs = self.findNode(toNode) # add to node add_label(label) if len(nodeIDs) > 1: # print("we got more target nodes with that name") # return raise MultipleNodeFoundError( "Multiple target nodes with this name") if len(nodeIDs) == 1: # one node exists toNodeID = nodeIDs[0] else: # no node exists, create new node toNodeID = self.createNewNode(toNode) tx = self.begin(autocommit=False) newRelationship = py2neo.Relationship(self.node(fromNodeID), relation, self.node(toNodeID)) tx.create(newRelationship) tx.commit() except MultipleNodeFoundError as problem: print("Multiple nodes found problem: {0}".format(problem))
def load_plays(user, artist, playcount): """Stores a user's plays of an artist in neo4j Args: user (str): lastfm user name artist (dict): lastfm artist """ user_node = GRAPH.find_one('User', property_key='name', property_value=user) artist_node = GRAPH.find_one('Artist', property_key='url', property_value=artist['url']) plays = GRAPH.match_one(start_node=user_node, rel_type='PLAYS', end_node=artist_node) if plays is None: plays = py2neo.Relationship(user_node, 'PLAYS', artist_node, **{'plays': playcount}) GRAPH.create(plays) else: plays['playcount'] += playcount GRAPH.push(plays)
def create_user_similarity(from_user_id: str, to_user_id: str, tdna_conf: float) -> bool: """ to create a relationship called "USER-SIMILAR" between two User nodes with confidence score as a property for that relationship. Args: from_user_id (string), from-node's user id to_user_id (string), to-node's user id tdna_conf (float), confidence score of similarity between users Returns: boolean, representing whether the relationship got inserted or not. """ matcher = py2neo.NodeMatcher(graph) from_node = matcher.match(USER_NODE_LABEL, user_id=from_user_id).first() to_node = matcher.match(USER_NODE_LABEL, user_id=to_user_id).first() user_similar = py2neo.Relationship(from_node, USER_SIMILAR_RELATIONSHIP, to_node, tdna_conf=tdna_conf) graph.create(user_similar) return graph.exists(user_similar)
def create_verified_similarity(from_user_id: str, verified_tweet_id: str, tdna_conf: float) -> bool: """ to create a relationship called "VERIFIED" between a legit User to his typing DNA verified tweet & confidence score as a property for that relationship. Args: from_user_id (string), person who tweeted, their user id verified_tweet_id (string), the authenticated tweet id, from that user id tdna_conf (float), confidence score of similarity between onboard & tweet pattern Returns: boolean, representing whether the relationship got inserted or not. """ matcher = py2neo.NodeMatcher(graph) from_node = matcher.match(USER_NODE_LABEL, user_id=from_user_id).first() to_node = matcher.match(TWEET_NODE_LABEL, tweet_id=verified_tweet_id).first() verified_similar = py2neo.Relationship(from_node, VERIFIED_RELATIONSHIP, to_node, tdna_conf=tdna_conf) graph.create(verified_similar) return graph.exists(verified_similar)
def create_tweet_impostor_similarity(impostor_user_id: str, unverified_tweet_id: str, tdna_conf: float) -> bool: """ to create a relationship called "TWEET-SIMILAR" between a fraudulent not-verified Tweet with the impostor User among the network (thus Among Us) & confidence score as a property for that relationship. Args: impostor_user_id (string), impostor user id whose onboard pattern matched the tweet. unverified_tweet_id (string), the unverified tweet, which was posted to dupe or fake. tdna_conf (float), confidence score of similarity between impostor onboard & tweet patterns Returns: boolean, representing whether the relationship got inserted or not. """ matcher = py2neo.NodeMatcher(graph) from_node = matcher.match(USER_NODE_LABEL, user_id=impostor_user_id).first() to_node = matcher.match(TWEET_NODE_LABEL, tweet_id=unverified_tweet_id).first() tweet_similar = py2neo.Relationship(from_node, TWEET_SIMILAR_RELATIONSHIP, to_node, tdna_conf=tdna_conf) graph.create(tweet_similar) return graph.exists(tweet_similar)
def make_systems_in_graph(settings, user_id, system_data): graph = social_graph(settings) system_owner = graph.find_one("User", "sql_id", user_id) if system_owner is None: print("user is not in graph yet (never logged in), skipping") return for entry in system_data: print(entry) try: systemNode = py2neo.Node( "System", system_id=entry['id'], system_uid=entry['system_uid'], name=entry['name'], description=entry['name'], location_lat=to_float(entry['location_lat']), location_lng=to_float(entry['location_lng']), status=entry['status'], creation_time=timestamp(), modified_time=timestamp()) graph.create(systemNode) relationship = py2neo.Relationship(system_owner, "SYS_ADMIN", systemNode) graph.create(relationship) except Exception as e: tb.print_exc() raise e
def export_neo4j(graph, uri, node_queue=200, edge_queue=5, show_progress=False): """Export hetnet to neo4j""" if show_progress: from tqdm import tqdm py2neo, _ = import_py2neo() if isinstance(uri, py2neo.Graph): db_graph = uri else: db_graph = py2neo.Graph(uri) # Delete all existing nodes db_graph.delete_all() # Create uniqueness constrains and indexes for metanode in graph.metagraph.get_nodes(): label = metanode.neo4j_label if "identifier" not in db_graph.schema.get_uniqueness_constraints(label): db_graph.schema.create_uniqueness_constraint(label, "identifier") if "name" not in db_graph.schema.get_indexes(label): db_graph.schema.create_index(label, "name") # Create nodes creator = Creator(db_graph, node_queue) queue = graph.get_nodes() if show_progress: queue = tqdm(queue, total=graph.n_nodes, desc="Importing nodes") for node in queue: label = node.metanode.neo4j_label data = sanitize_data(node.data) neo_node = py2neo.Node( label, identifier=node.identifier, name=node.name, **data ) creator.append(neo_node) creator.create() # Create edges creator = Creator(db_graph, edge_queue) queue = graph.get_edges(exclude_inverts=True) if show_progress: queue = tqdm(queue, total=graph.n_edges, desc="Importing edges") for edge in queue: metaedge = edge.metaedge rel_type = metaedge.neo4j_rel_type source_label = metaedge.source.neo4j_label target_label = metaedge.target.neo4j_label source = db_graph.find_one(source_label, "identifier", edge.source.identifier) target = db_graph.find_one(target_label, "identifier", edge.target.identifier) data = sanitize_data(edge.data) neo_rel = py2neo.Relationship(source, rel_type, target, **data) creator.append(neo_rel) creator.create() return db_graph
def authority_photos(authority): query = authority.node.properties["person_name_heb"] print("Photo {} : {}".format(authority.node.properties["id"], query), flush=True) photos = Photos(query) for photo, _ in zip(photos, range(10)): portrait_node = create_entity(photo) graph.create_unique( py2neo.Relationship(authority.node, "subject_of", portrait_node))
def place_flows(nodes, graph, label="Sim", rate=0): chunks = split_in_pairs(nodes) for src, dst in chunks: flow = graph.match_one(start_node=src, rel_type=label, end_node=dst) if not flow: # print("no flow created yet") flow = py2neo.Relationship(src, label, dst, rate=0) graph.create_unique(flow) flow.properties['rate'] = flow.properties['rate'] + rate flow.push()
def _create_relationship(self, src_node, relationship_type, target_node, **kwargs): assert isinstance(src_node, py2neo.Node) # must create target_node if doesn't exist target_node = self._convert_to_target_node(relationship_type, target_node) relation = py2neo.Relationship(src_node, relationship_type.name, target_node, **kwargs) self.graph_db.create(relation) return relation
def addRelationship(eth, ipSrc, ipDst, tx, tabNode, linkInfo): pcSrc = getPcInfo(mac_addr(eth.src), ipSrc) pcDst = getPcInfo(mac_addr(eth.dst), ipDst) nodeSrc, rel = addNode(tx, tabNode, pcSrc) nbNew = rel nodeDst, rel = addNode(tx, tabNode, pcDst) nbNew += rel rs = py2neo.Relationship(nodeSrc, linkInfo, nodeDst) tx.create(rs) return nbNew
async def sync_modules(self): org_nodes: Dict[str, py2neo.Node] = {} user_nodes: Dict[str, py2neo.Node] = {} tx = self.graph_db.begin() self.logger.debug("orgs: %s", await self.auth_module.read_orgs()) for org in await self.auth_module.read_orgs(): # creating organisation org_node = tx.graph.nodes.match( "org", org_id=org["organisation_id"], ).first() if org_node is None: org_node = py2neo.Node( "org", org_id=org["organisation_id"], org_name=org["organisation_name"], ) tx.create(org_node) org_nodes[org["organisation_id"]] = org_node self.logger.debug("users: %s", await self.auth_module.read_users()) for user in await self.auth_module.read_users(): # creating user user_node = tx.graph.nodes.match( "user", user_id=user["user_id"], ).first() if user_node is None: user_node = py2neo.Node( "user", user_id=user["user_id"], user_name=user["user_name"], email=user["email"], loa=user["level_of_access"], ) tx.create(user_node) user_nodes[user["user_id"]] = user_node # connect org to user user2org_relation = tx.graph.relationships.match(nodes=[ org_nodes[user["member_of"]], user_node, ]).first() if user2org_relation is None: user2org_relation = py2neo.Relationship( org_nodes[user["member_of"]], "contains", user_node, ) tx.create(user2org_relation) tx.commit()
def _convert_relationship_to_edge(subject_node, relationship, object_node): relationship._prefix = "DR" attrs = { atr.atr_name: atr.atr_value for atr in relationship.get_attributes() } return p2n.Relationship(subject_node, relationship.rel_name, object_node, ui=relationship.ui, src=relationship.src, **attrs)
def _create_atom_nodes(concept): concept_node = ui_to_node[concept.ui] for atom in concept.get_atoms(): atom._prefix = "DA" atom_node = p2n.Node(f"{concept.concept_type}_ATOM", ui=atom.ui, name=atom.term, src=atom.src, src_id=atom.src_id, is_preferred=atom.is_preferred, **atom.attrs) graph.create( p2n.Relationship(concept_node, "has_synonym", atom_node))
def create_node(label, code, name, parent_code=None): node1 = graph.nodes.match(label, code=code).first() if node1: print("Node: " + name + " exists!") pass else: node = py2neo.Node(label, code=code, name=name) graph.create(node) print("Create Node: " + name) if parent_code: parent_node = graph.nodes.match(label, code=parent_code).first() #print(parent_node) rel = py2neo.Relationship(node, "属于", parent_node) graph.create(rel) print("Create Relationship between Node " + name + " and Node " + parent_node['name'])
def to_neo4j(graph, neo_connection, context=None): """Uploads a BEL graph to Neo4J graph database using :mod:`py2neo` :param BELGraph graph: A BEL Graph :param neo_connection: A :mod:`py2neo` connection object. Refer to the `py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_ for how to build this object. :type neo_connection: :class:`py2neo.Graph` :param str context: A disease context to allow for multiple disease models in one neo4j instance. Each edge will be assigned an attribute :code:`pybel_context` with this value Example Usage: >>> import pybel, py2neo >>> url = 'http://resource.belframework.org/belframework/1.0/knowledge/small_corpus.bel' >>> g = pybel.from_url(url) >>> neo_graph = py2neo.Graph("http://localhost:7474/db/data/") # use your own connection settings >>> pybel.to_neo4j(g, neo_graph) """ import py2neo tx = neo_connection.begin() node_map = {} for node, data in graph.nodes(data=True): node_type = data[FUNCTION] attrs = {k: v for k, v in data.items() if k not in {FUNCTION, NAME}} attrs['name'] = calculate_canonical_name(graph, node) if NAME in data: attrs['identifier'] = data[NAME] node_map[node] = py2neo.Node(node_type, bel=node_to_bel(data), **attrs) tx.create(node_map[node]) for u, v, data in graph.edges(data=True): rel_type = data[RELATION] attrs = flatten_dict(data) if context is not None: attrs[PYBEL_CONTEXT_TAG] = str(context) rel = py2neo.Relationship(node_map[u], rel_type, node_map[v], **attrs) tx.create(rel) tx.commit()
def export_neo4j(graph, uri, node_queue=100, edge_queue=100): """Export hetnet to neo4j""" db_graph = py2neo.Graph(uri) # Delete all existing nodes db_graph.delete_all() # Create uniqueness constrains and indexes for metanode in graph.metagraph.get_nodes(): label = as_label(metanode) if 'identifier' not in db_graph.schema.get_uniqueness_constraints( label): db_graph.schema.create_uniqueness_constraint(label, 'identifier') if 'name' not in db_graph.schema.get_indexes(label): db_graph.schema.create_index(label, 'name') # Create nodes creator = Creator(db_graph, node_queue) for node in graph.get_nodes(): label = as_label(node.metanode) data = sanitize_data(node.data) neo_node = py2neo.Node(label, identifier=node.identifier, name=node.name, **data) creator.append(neo_node) creator.create() # Create edges creator = Creator(db_graph, edge_queue) for edge in graph.get_edges(exclude_inverts=True): metaedge = edge.metaedge rel_type = as_type(metaedge) source_label = as_label(metaedge.source) target_label = as_label(metaedge.target) source = db_graph.find_one(source_label, 'identifier', edge.source.identifier) target = db_graph.find_one(target_label, 'identifier', edge.target.identifier) data = sanitize_data(edge.data) neo_rel = py2neo.Relationship(source, rel_type, target, **data) creator.append(neo_rel) creator.create() return db_graph
def get_tasks(self): tasks = [] # relationships = [] for f in self.followings: if f['hashid']: dst_node = self.merge_node(f) follows = py2neo.Relationship(self.src_node, 'FOLLOWS', dst_node) # relationships.append(follows) self.g.create_unique(follows) if not rh.is_user_crawled(f['domain']): tasks.append(f['domain'].encode('utf-8')) # if len(relationships) > 0: # self.g.create_unique(*relationships) return tasks
def update(self): """Updates all hosts' and connections' HMMs using the forward algorithm and the given evidence up to this point, then updates the database""" # First process the newly received evidence self.process_evidence(self.evidence) # Apply forward algorithm for h in self.hosts: self.hosts[h].update() for c in self.connections: self.connections[c].update() # c is connection tuple consisting of two network address + port tuples # Extract the source and destination hosts' network address tuples src = c[0] dst = c[1] # Raw address string of form 3.3.3.3:21 that will be used as a unique identifier in the database src_raw = str(c[0][0]) + ":" + str(c[0][1]) dst_raw = str(c[1][0]) + ":" + str(c[1][1]) # Fetch the source and destination hosts' HMMs, then src_hmm.p is the probability the source host exists given the evidence up to this point src_hmm = self.hosts[src] dst_hmm = self.hosts[dst] if src[0] == "8.8.8.8": print("p = ", src_hmm.p) # Now get the probability the connection exists conn_p = self.connections[c].p # Create nodes and a relationship to insert into the database src_node = neo.Node("Host", addr=src_raw, ip=src[0], port=src[1], p=src_hmm.p[0]) dst_node = neo.Node("Host", addr=dst_raw, ip=dst[0], port=dst[1], p=dst_hmm.p[0]) connection = neo.Relationship(src_node, "CONNECTED", dst_node, p=conn_p[0]) neo.Graph.merge(graph, connection, "Host", "addr")
def load_friendship(user, friend): """Stores a lastfm user's friend in neo4j Args: user (str): lastfm user name friend (str): lastfm user name """ user_node = GRAPH.find_one('User', property_key='name', property_value=user) friend_node = GRAPH.find_one('User', property_key='name', property_value=friend) friendship = GRAPH.match_one(start_node=user_node, rel_type='FRIENDS', end_node=friend_node, bidirectional=True) if friendship is None: friendship = py2neo.Relationship(user_node, 'FRIENDS', friend_node) GRAPH.create(friendship)
def write_neo(nodes, edges): if not common.settings.neo4j.getboolean("Available"): return print("writing graph to neo") graph = get_graph() graph.delete_all() transaction = graph.begin() count = 0 for f, t, data in tqdm(edges): count += 1 node_from = py2neo.Node(nodes[f]["label"], **{**nodes[f], **{"id_str": f}}) node_to = py2neo.Node(nodes[t]["label"], **{**nodes[t], **{"id_str": t}}) edge = py2neo.Relationship(node_from, data["label"], node_to, **data) transaction.merge(edge) if count == common.settings.neo4j.getint("TransactionsBeforeCommit"): count = 0 transaction.commit() transaction = graph.begin() transaction.commit()
async def create_dict( self, user_id: str, dict_id: str, words: List[str], private: bool = False, name: Optional[str] = None, has_access: Optional[List[str]] = None, ) -> Dict[str, Any]: self.logger.debug("adding new document") tx = self.graph_db.begin() # check that Document with the same id dict_with_same_name = tx.graph.nodes.match( "Dictionary", dict_id=dict_id, ).first() if dict_with_same_name is not None: raise DictNameError # create Dict dict_node = py2neo.Node( "Dictionary", dict_id=dict_id, words=words, private=private, name=name, ) tx.create(dict_node) # connect Dict with creator author = tx.graph.nodes.match("user", user_id=user_id).first() tx.create(py2neo.Relationship(author, "created", dict_node)) tx.commit()
def run(xconfig): """ build graph in Neo4j based on the Kanji dataset """ # check for extra options margs = xconfig.run.modargs # connect to Mongo mgx = mongo(xconfig.mongo.uri) # get all kanji from Mongo kset = mgx.find("kanji", {}) logthis("** Kanji objects:", suffix=len(kset), loglevel=LL.INFO) # connect to Neo4j try: neo = py2neo.Graph(xconfig.neo4j.uri) ncount = neo.cypher.execute('MATCH (n) RETURN count(*)') except Exception as e: logexc(e, "Failed to connect to Neo4j dataset") failwith(ER.PROCFAIL, "Unable to continue. Aborting.") # if 'clear' is passed as an extra parg, then drop all existing nodes/rels if 'clear' in margs: logthis("Deleting existing data...", loglevel=LL.INFO) neo.cypher.execute("MATCH (n) DETACH DELETE n") # create node constraints logthis("Creating constraints...", loglevel=LL.VERBOSE) neo.cypher.execute( "CREATE CONSTRAINT ON (k:Kanji) ASSERT k.kanji IS UNIQUE") neo.cypher.execute( "CREATE CONSTRAINT ON (r:Radical) ASSERT r.radical IS UNIQUE") neo.cypher.execute( "CREATE CONSTRAINT ON (s:Sense) ASSERT s.sense IS UNIQUE") neo.cypher.execute( "CREATE CONSTRAINT ON (r:Reading) ASSERT r.reading IS UNIQUE") neo.cypher.execute("CREATE CONSTRAINT ON (g:Joyo) ASSERT g.joyo IS UNIQUE") neo.cypher.execute("CREATE CONSTRAINT ON (g:Jlpt) ASSERT g.jlpt IS UNIQUE") neo.cypher.execute("CREATE CONSTRAINT ON (k:Skip) ASSERT k.skip IS UNIQUE") # Build nodes & relationships logthis("** Building graph...", loglevel=LL.INFO) for kk, tk in kset.iteritems(): logthis(">>>------[ %5d ] Kanji node <%s> -----" % (kk, tk['kanji']), loglevel=LL.DEBUG) # Kanji try: freq = int(tk['freq']) except: freq = 0 knode = py2neo.Node("Kanji", kanji=tk['kanji'], ucs=tk['_id'], freq=freq) # Radicals xnodes = [] xrels = [] if tk.has_key('xrad') and len(tk['xrad']) > 0: for tr, tv in tk['xrad'].iteritems(): # check if a radical exists in db.radical rrad = mgx.findOne("radical", {"radical": tr}) xrad = {} if rrad: xrad = { "rad_id": rrad['_id'], "alt": rrad['alt'], "radname": rrad['radname']['ja'], "radname_en": rrad['radname']['en'] } else: rrad = mgx.findOne("kanji", {"kanji": tr}) if rrad: # Created Kanji-Kanji relationship xrad = False try: freq = int(rrad['freq']) except: freq = 0 xnodes.append( py2neo.Node("Kanji", kanji=rrad['kanji'], ucs=rrad['_id'], freq=freq)) xrels.append( py2neo.Relationship(knode, "CONTAINS", xnodes[-1], position=tv.get( 'position', None))) else: xrad = {"non_standard": True} if xrad: xnodes.append(py2neo.Node("Radical", radical=tr, **xrad)) xrels.append( py2neo.Relationship(knode, "CONTAINS", xnodes[-1], position=tv.get('position', None))) elif tk.has_key('krad'): for tr in tk['krad']: # check if a radical exists in db.radical rrad = mgx.findOne("radical", {"radical": tr}) xrad = {} if rrad: xrad = { "rad_id": rrad['_id'], "alt": rrad['alt'], "radname": rrad['radname']['ja'], "radname_en": rrad['radname']['en'] } else: rrad = mgx.findOne("kanji", {"kanji": tr}) if rrad: # Created Kanji-Kanji relationship xrad = False try: freq = int(rrad['freq']) except: freq = 0 xnodes.append( py2neo.Node("Kanji", kanji=rrad['kanji'], ucs=rrad['_id'], freq=freq)) xrels.append( py2neo.Relationship(knode, "CONTAINS", xnodes[-1])) else: xrad = {"non_standard": True} if xrad: xnodes.append(py2neo.Node("Radical", radical=tr, **xrad)) xrels.append( py2neo.Relationship(knode, "CONTAINS", xnodes[-1])) # Senses if tk.has_key('meaning') and tk['meaning'].get('en'): for ts in tk['meaning']['en']: xnodes.append(py2neo.Node("Sense", sense=ts, lang="en")) xrels.append(py2neo.Relationship(knode, "MEANS", xnodes[-1])) # Readings (on-yomi, kun-yomi, nanori) if tk.has_key('reading'): if tk['reading'].has_key('ja_on'): for tr in tk['reading']['ja_on']: xnodes.append(py2neo.Node("Reading", reading=tr)) xrels.append( py2neo.Relationship(knode, "READS", xnodes[-1], yomi="on")) if tk['reading'].has_key('ja_kun'): for tr in tk['reading']['ja_kun']: xnodes.append(py2neo.Node("Reading", reading=tr)) xrels.append( py2neo.Relationship(knode, "READS", xnodes[-1], yomi="kun")) if tk['reading'].has_key('nanori'): for tr in tk['reading']['nanori']: xnodes.append(py2neo.Node("Reading", reading=tr)) xrels.append( py2neo.Relationship(knode, "READS", xnodes[-1], yomi="nanori")) # Joyo if tk.has_key('grade') and tk.has_key('jindex'): xnodes.append(py2neo.Node("Joyo", joyo=int(tk['grade']))) xrels.append( py2neo.Relationship(xnodes[-1], "SUBSET", knode, jindex=tk['jindex'])) # JLPT if tk.has_key('jlpt') and isinstance(tk['jlpt'], int): xnodes.append(py2neo.Node("Jlpt", jlpt=int(tk['jlpt']))) xrels.append(py2neo.Relationship(xnodes[-1], "SUBSET", knode)) # SKIP if tk.has_key('qcode') and tk['qcode'].has_key('skip'): xnodes.append(py2neo.Node("Skip", skip=tk['qcode']['skip'])) xrels.append(py2neo.Relationship(knode, "WRITTEN", xnodes[-1])) # Create Kanji node try: neo.create(knode) except Exception as e: logexc(e, u'Failed to create Kanji node') # Create nodes for tnode in xnodes: try: neo.create(tnode) except Exception as e: logexc(e, u'Failed to create aux node') # Build relations for trel in xrels: # Check if Nodes are bound sn = trel.start_node en = trel.end_node # if start node is not bound, then attempt a lookup if not sn.bound: nlab = list(sn.labels)[0] nsn = neo.find_one(nlab, nlab.lower(), sn[nlab.lower()]) if nsn: logthis(">>> Xref OK: %s '%s'" % (nlab, sn[nlab.lower()]), loglevel=LL.DEBUG) sn = nsn # if end node is not bound, then attempt a lookup if not en.bound: elab = list(en.labels)[0] nen = neo.find_one(elab, elab.lower(), en[elab.lower()]) if nen: logthis(">>> Xref OK: %s '%s'" % (elab, en[elab.lower()]), loglevel=LL.DEBUG) en = nen # Rebuild relationship rrel = py2neo.Relationship(sn, trel.type, en, **trel.properties) try: neo.create_unique(rrel) except Exception as e: logexc(e, "Failed to build relationship")
def to_neo4j(graph, neo_connection, use_tqdm=False): """Upload a BEL graph to a Neo4j graph database using :mod:`py2neo`. :param pybel.BELGraph graph: A BEL Graph :param neo_connection: A :mod:`py2neo` connection object. Refer to the `py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_ for how to build this object. :type neo_connection: str or py2neo.Graph Example Usage: >>> import py2neo >>> import pybel >>> from pybel.examples import sialic_acid_graph >>> neo_graph = py2neo.Graph("http://localhost:7474/db/data/") # use your own connection settings >>> pybel.to_neo4j(sialic_acid_graph, neo_graph) """ import py2neo if isinstance(neo_connection, str): neo_connection = py2neo.Graph(neo_connection) tx = neo_connection.begin() node_map = {} nodes = list(graph) if use_tqdm: nodes = tqdm(nodes, desc='nodes') for node in nodes: if NAMESPACE not in node or VARIANTS in node or MEMBERS in node or FUSION in node: attrs = {'name': node.as_bel()} else: attrs = {'namespace': node.namespace} if node.name and node.identifier: attrs['name'] = node.name attrs['identifier'] = node.identifier elif node.identifier and not node.name: attrs['name'] = node.identifier elif node.name and not node.identifier: attrs['name'] = node.name node_map[node] = py2neo.Node(node.function, **attrs) tx.create(node_map[node]) edges = graph.edges(keys=True, data=True) if use_tqdm: edges = tqdm(edges, desc='edges') for u, v, key, node in edges: rel_type = node[RELATION] d = node.copy() del d[RELATION] attrs = {} annotations = d.pop(ANNOTATIONS, None) if annotations: for annotation, values in annotations.items(): attrs[annotation] = list(values) citation = d.pop(CITATION, None) if citation: attrs[CITATION] = '{}:{}'.format(citation[CITATION_TYPE], citation[CITATION_REFERENCE]) if EVIDENCE in d: attrs[EVIDENCE] = d[EVIDENCE] for side in (SUBJECT, OBJECT): side_data = d.get(side) if side_data: attrs.update(flatten_dict(side_data, parent_key=side)) rel = py2neo.Relationship(node_map[u], rel_type, node_map[v], key=key, **attrs) tx.create(rel) tx.commit()