def processItem(item):
neo4jGraph, topologicGraph, labelKey, relationshipKey, bidirectional, deleteAll, tolerance, run = item
if not (run):
return None
import time
gmt = time.gmtime()
timestamp = str(gmt.tm_zone) + "_" + str(gmt.tm_year) + "_" + str(
gmt.tm_mon) + "_" + str(gmt.tm_wday) + "_" + str(
gmt.tm_hour) + "_" + str(gmt.tm_min) + "_" + str(gmt.tm_sec)
vertices = []
_ = topologicGraph.Vertices(vertices)
edges = []
_ = topologicGraph.Edges(edges)
notUsed = []
tx = neo4jGraph.begin()
nodes = []
for i in range(len(vertices)):
vDict = vertices[i].GetDictionary()
keys, values = getKeysAndValues(vDict)
keys.append("x")
keys.append("y")
keys.append("z")
keys.append("timestamp")
keys.append("location")
values.append(vertices[i].X())
values.append(vertices[i].Y())
values.append(vertices[i].Z())
values.append(timestamp)
values.append(
sp.CartesianPoint(
[vertices[i].X(), vertices[i].Y(), vertices[i].Z()]))
zip_iterator = zip(keys, values)
pydict = dict(zip_iterator)
if (labelKey == 'None') or (not (labelKey)):
nodeName = "TopologicGraphVertex"
else:
nodeName = str(getValueAtKey(vDict, labelKey))
n = py2neo.Node(nodeName, **pydict)
tx.create(n)
nodes.append(n)
for i in range(len(edges)):
e = edges[i]
sv = e.StartVertex()
ev = e.EndVertex()
sn = nodes[vertexIndex(sv, vertices, tolerance)]
en = nodes[vertexIndex(ev, vertices, tolerance)]
ed = e.GetDictionary()
relationshipType = getValueAtKey(ed, relationshipKey)
if not (relationshipType):
relationshipType = "Connected To"
snen = py2neo.Relationship(sn, relationshipType, en)
tx.create(snen)
if bidirectional:
snen = py2neo.Relationship(en, relationshipType, sn)
tx.create(snen)
if deleteAll:
neo4jGraph.delete_all()
neo4jGraph.commit(tx)
return neo4jGraph
def create_followers(node):
"""
为参数结点在数据库中生成follower
:param node: label为developer的结点
:return:
"""
info = {}
try:
info = requests.get(node["followers_url"], headers=headers).json()
except KeyError:
print("Node error")
exit(-1)
if info:
followers = [member["url"] for member in info]
for follower in followers:
follower_info = requests.get(follower, headers=headers).json()
properties, relationships = get_developer_info(follower_info)
temp = properties.copy()
temp.update(relationships)
follower_node = py2neo.Node("DEVELOPER", **temp)
rel1 = py2neo.Relationship(node, "IS_FOLLOWED_BY", follower_node)
rel2 = py2neo.Relationship(follower_node, "FOLLOWS", node)
tx.merge(follower_node, "DEVELOPER", "id")
tx.create(rel1)
tx.create(rel2)
def authority_portrait(authority):
query = authority.node.properties["person_name_absolute"]
print("Portrait {} : {}".format(authority.node.properties["id"], query),
flush=True)
portraits = Portraits(query)
for portrait in portraits:
portrait_node = create_entity(portrait)
graph.create_unique(
py2neo.Relationship(authority.node, "subject_of", portrait_node))
graph.create_unique(
py2neo.Relationship(authority.node, "portrait_of", portrait_node))
def processItem(item):
import time
gmt = time.gmtime()
timestamp = str(gmt.tm_zone)+"_"+str(gmt.tm_year)+"_"+str(gmt.tm_mon)+"_"+str(gmt.tm_wday)+"_"+str(gmt.tm_hour)+"_"+str(gmt.tm_min)+"_"+str(gmt.tm_sec)
neo4jGraph, topologicGraph, categoryKey, tolerance = item
vertices = []
_ = topologicGraph.Vertices(vertices)
edges = []
_ = topologicGraph.Edges(edges)
notUsed = []
tx = neo4jGraph.begin()
nodes = []
for i in range(len(vertices)):
vDict = vertices[i].GetDictionary()
keys, values = getKeysAndValues(vDict)
keys.append("x")
keys.append("y")
keys.append("z")
keys.append("timestamp")
keys.append("location")
values.append(vertices[i].X())
values.append(vertices[i].Y())
values.append(vertices[i].Z())
values.append(timestamp)
values.append(sp.CartesianPoint([vertices[i].X(),vertices[i].Y(),vertices[i].Z()]))
zip_iterator = zip(keys, values)
pydict = dict(zip_iterator)
if categoryKey == 'None':
nodeName = "TopologicGraphVertex"
else:
nodeName = str(values[keys.index(categoryKey)])
n = py2neo.Node(nodeName, **pydict)
neo4jGraph.cypher.execute("CREATE INDEX FOR (n:%s) on (n.name)" %
n.nodelabel)
#try:
#neo4jGraph.cypher.execute("CREATE INDEX FOR (n:%s) on (n.name)" %
#n.nodelabel)
#except:
#pass
tx.create(n)
nodes.append(n)
for i in range(len(edges)):
e = edges[i]
sv = e.StartVertex()
ev = e.EndVertex()
sn = nodes[vertexIndex(sv, vertices, tolerance)]
en = nodes[vertexIndex(ev, vertices, tolerance)]
snen = py2neo.Relationship(sn, "CONNECTEDTO", en)
tx.create(snen)
snen = py2neo.Relationship(en, "CONNECTEDTO", sn)
tx.create(snen)
neo4jGraph.commit(tx)
return neo4jGraph
def create_authentication_failed(from_user_id: str,
unverified_tweet_id: str) -> bool:
"""
to create a relationship called "UNVERIFIED" between a legit User
to an impostor tweet from their account.
Args:
from_user_id (string), person who tweeted, their user id
unverified_tweet_id (string), the unauthenticated tweet id, from that user id
Returns:
boolean, representing whether the relationship got inserted or not.
"""
matcher = py2neo.NodeMatcher(graph)
from_node = matcher.match(USER_NODE_LABEL, user_id=from_user_id).first()
to_node = matcher.match(TWEET_NODE_LABEL,
tweet_id=unverified_tweet_id).first()
failed_auth = py2neo.Relationship(
from_node,
UNVERIFIED_RELATIONSHIP,
to_node,
)
graph.create(failed_auth)
return graph.exists(failed_auth)
def create_relation(self,
source,
dest,
properties={},
type_=BaseGraphDB.DEFAULT_RELATION):
log.log.debug(
'Creating %s from %s to %s with properties %s',
type_.name,
source.properties['name'],
dest.properties['name'],
properties
)
s = self.get(
source.properties['name'],
collection=source.collection,
db_convert=False
)
d = self.get(
dest.properties['name'],
collection=dest.collection,
db_convert=False
)
r = py2neo.Relationship(s, type_.name, d, **properties)
self._r.create(r)
return r
def addRelation(self, fromNode, toNode, relation):
"""Adds a new relationship between nodes with label."""
try:
nodeIDs = self.findNode(fromNode)
# add to node add_label(label)
if len(nodeIDs) > 1:
# print("we got more source nodes with that name")
# return
raise MultipleNodeFoundError(
"Multiple source nodes with this name")
if len(nodeIDs) == 1: # one node exists
fromNodeID = nodeIDs[0]
else: # no node exists, create new node
fromNodeID = self.createNewNode(fromNode)
nodeIDs = self.findNode(toNode)
# add to node add_label(label)
if len(nodeIDs) > 1:
# print("we got more target nodes with that name")
# return
raise MultipleNodeFoundError(
"Multiple target nodes with this name")
if len(nodeIDs) == 1: # one node exists
toNodeID = nodeIDs[0]
else: # no node exists, create new node
toNodeID = self.createNewNode(toNode)
tx = self.begin(autocommit=False)
newRelationship = py2neo.Relationship(self.node(fromNodeID),
relation,
self.node(toNodeID))
tx.create(newRelationship)
tx.commit()
except MultipleNodeFoundError as problem:
print("Multiple nodes found problem: {0}".format(problem))
def load_plays(user, artist, playcount):
"""Stores a user's plays of an artist in neo4j
Args:
user (str): lastfm user name
artist (dict): lastfm artist
"""
user_node = GRAPH.find_one('User',
property_key='name',
property_value=user)
artist_node = GRAPH.find_one('Artist',
property_key='url',
property_value=artist['url'])
plays = GRAPH.match_one(start_node=user_node,
rel_type='PLAYS',
end_node=artist_node)
if plays is None:
plays = py2neo.Relationship(user_node, 'PLAYS', artist_node,
**{'plays': playcount})
GRAPH.create(plays)
else:
plays['playcount'] += playcount
GRAPH.push(plays)
def create_user_similarity(from_user_id: str, to_user_id: str,
tdna_conf: float) -> bool:
"""
to create a relationship called "USER-SIMILAR" between two User nodes with
confidence score as a property for that relationship.
Args:
from_user_id (string), from-node's user id
to_user_id (string), to-node's user id
tdna_conf (float), confidence score of similarity between users
Returns:
boolean, representing whether the relationship got inserted or not.
"""
matcher = py2neo.NodeMatcher(graph)
from_node = matcher.match(USER_NODE_LABEL, user_id=from_user_id).first()
to_node = matcher.match(USER_NODE_LABEL, user_id=to_user_id).first()
user_similar = py2neo.Relationship(from_node,
USER_SIMILAR_RELATIONSHIP,
to_node,
tdna_conf=tdna_conf)
graph.create(user_similar)
return graph.exists(user_similar)
def create_verified_similarity(from_user_id: str, verified_tweet_id: str,
tdna_conf: float) -> bool:
"""
to create a relationship called "VERIFIED" between a legit User
to his typing DNA verified tweet & confidence score as a
property for that relationship.
Args:
from_user_id (string), person who tweeted, their user id
verified_tweet_id (string), the authenticated tweet id, from that user id
tdna_conf (float), confidence score of similarity between onboard & tweet pattern
Returns:
boolean, representing whether the relationship got inserted or not.
"""
matcher = py2neo.NodeMatcher(graph)
from_node = matcher.match(USER_NODE_LABEL, user_id=from_user_id).first()
to_node = matcher.match(TWEET_NODE_LABEL,
tweet_id=verified_tweet_id).first()
verified_similar = py2neo.Relationship(from_node,
VERIFIED_RELATIONSHIP,
to_node,
tdna_conf=tdna_conf)
graph.create(verified_similar)
return graph.exists(verified_similar)
def create_tweet_impostor_similarity(impostor_user_id: str,
unverified_tweet_id: str,
tdna_conf: float) -> bool:
"""
to create a relationship called "TWEET-SIMILAR" between a fraudulent
not-verified Tweet with the impostor User among the network (thus Among Us) &
confidence score as a property for that relationship.
Args:
impostor_user_id (string), impostor user id whose onboard pattern matched the tweet.
unverified_tweet_id (string), the unverified tweet, which was posted to dupe or fake.
tdna_conf (float), confidence score of similarity between impostor onboard & tweet patterns
Returns:
boolean, representing whether the relationship got inserted or not.
"""
matcher = py2neo.NodeMatcher(graph)
from_node = matcher.match(USER_NODE_LABEL,
user_id=impostor_user_id).first()
to_node = matcher.match(TWEET_NODE_LABEL,
tweet_id=unverified_tweet_id).first()
tweet_similar = py2neo.Relationship(from_node,
TWEET_SIMILAR_RELATIONSHIP,
to_node,
tdna_conf=tdna_conf)
graph.create(tweet_similar)
return graph.exists(tweet_similar)
def make_systems_in_graph(settings, user_id, system_data):
graph = social_graph(settings)
system_owner = graph.find_one("User", "sql_id", user_id)
if system_owner is None:
print("user is not in graph yet (never logged in), skipping")
return
for entry in system_data:
print(entry)
try:
systemNode = py2neo.Node(
"System",
system_id=entry['id'],
system_uid=entry['system_uid'],
name=entry['name'],
description=entry['name'],
location_lat=to_float(entry['location_lat']),
location_lng=to_float(entry['location_lng']),
status=entry['status'],
creation_time=timestamp(),
modified_time=timestamp())
graph.create(systemNode)
relationship = py2neo.Relationship(system_owner, "SYS_ADMIN",
systemNode)
graph.create(relationship)
except Exception as e:
tb.print_exc()
raise e
def export_neo4j(graph, uri, node_queue=200, edge_queue=5, show_progress=False):
"""Export hetnet to neo4j"""
if show_progress:
from tqdm import tqdm
py2neo, _ = import_py2neo()
if isinstance(uri, py2neo.Graph):
db_graph = uri
else:
db_graph = py2neo.Graph(uri)
# Delete all existing nodes
db_graph.delete_all()
# Create uniqueness constrains and indexes
for metanode in graph.metagraph.get_nodes():
label = metanode.neo4j_label
if "identifier" not in db_graph.schema.get_uniqueness_constraints(label):
db_graph.schema.create_uniqueness_constraint(label, "identifier")
if "name" not in db_graph.schema.get_indexes(label):
db_graph.schema.create_index(label, "name")
# Create nodes
creator = Creator(db_graph, node_queue)
queue = graph.get_nodes()
if show_progress:
queue = tqdm(queue, total=graph.n_nodes, desc="Importing nodes")
for node in queue:
label = node.metanode.neo4j_label
data = sanitize_data(node.data)
neo_node = py2neo.Node(
label, identifier=node.identifier, name=node.name, **data
)
creator.append(neo_node)
creator.create()
# Create edges
creator = Creator(db_graph, edge_queue)
queue = graph.get_edges(exclude_inverts=True)
if show_progress:
queue = tqdm(queue, total=graph.n_edges, desc="Importing edges")
for edge in queue:
metaedge = edge.metaedge
rel_type = metaedge.neo4j_rel_type
source_label = metaedge.source.neo4j_label
target_label = metaedge.target.neo4j_label
source = db_graph.find_one(source_label, "identifier", edge.source.identifier)
target = db_graph.find_one(target_label, "identifier", edge.target.identifier)
data = sanitize_data(edge.data)
neo_rel = py2neo.Relationship(source, rel_type, target, **data)
creator.append(neo_rel)
creator.create()
return db_graph
def authority_photos(authority):
query = authority.node.properties["person_name_heb"]
print("Photo {} : {}".format(authority.node.properties["id"], query),
flush=True)
photos = Photos(query)
for photo, _ in zip(photos, range(10)):
portrait_node = create_entity(photo)
graph.create_unique(
py2neo.Relationship(authority.node, "subject_of", portrait_node))
def place_flows(nodes, graph, label="Sim", rate=0):
chunks = split_in_pairs(nodes)
for src, dst in chunks:
flow = graph.match_one(start_node=src, rel_type=label, end_node=dst)
if not flow:
# print("no flow created yet")
flow = py2neo.Relationship(src, label, dst, rate=0)
graph.create_unique(flow)
flow.properties['rate'] = flow.properties['rate'] + rate
flow.push()
def _create_relationship(self, src_node, relationship_type, target_node,
**kwargs):
assert isinstance(src_node, py2neo.Node)
# must create target_node if doesn't exist
target_node = self._convert_to_target_node(relationship_type,
target_node)
relation = py2neo.Relationship(src_node, relationship_type.name,
target_node, **kwargs)
self.graph_db.create(relation)
return relation
def addRelationship(eth, ipSrc, ipDst, tx, tabNode, linkInfo):
pcSrc = getPcInfo(mac_addr(eth.src), ipSrc)
pcDst = getPcInfo(mac_addr(eth.dst), ipDst)
nodeSrc, rel = addNode(tx, tabNode, pcSrc)
nbNew = rel
nodeDst, rel = addNode(tx, tabNode, pcDst)
nbNew += rel
rs = py2neo.Relationship(nodeSrc, linkInfo, nodeDst)
tx.create(rs)
return nbNew
async def sync_modules(self):
org_nodes: Dict[str, py2neo.Node] = {}
user_nodes: Dict[str, py2neo.Node] = {}
tx = self.graph_db.begin()
self.logger.debug("orgs: %s", await self.auth_module.read_orgs())
for org in await self.auth_module.read_orgs():
# creating organisation
org_node = tx.graph.nodes.match(
"org",
org_id=org["organisation_id"],
).first()
if org_node is None:
org_node = py2neo.Node(
"org",
org_id=org["organisation_id"],
org_name=org["organisation_name"],
)
tx.create(org_node)
org_nodes[org["organisation_id"]] = org_node
self.logger.debug("users: %s", await self.auth_module.read_users())
for user in await self.auth_module.read_users():
# creating user
user_node = tx.graph.nodes.match(
"user",
user_id=user["user_id"],
).first()
if user_node is None:
user_node = py2neo.Node(
"user",
user_id=user["user_id"],
user_name=user["user_name"],
email=user["email"],
loa=user["level_of_access"],
)
tx.create(user_node)
user_nodes[user["user_id"]] = user_node
# connect org to user
user2org_relation = tx.graph.relationships.match(nodes=[
org_nodes[user["member_of"]],
user_node,
]).first()
if user2org_relation is None:
user2org_relation = py2neo.Relationship(
org_nodes[user["member_of"]],
"contains",
user_node,
)
tx.create(user2org_relation)
tx.commit()
def _convert_relationship_to_edge(subject_node, relationship, object_node):
relationship._prefix = "DR"
attrs = {
atr.atr_name: atr.atr_value
for atr in relationship.get_attributes()
}
return p2n.Relationship(subject_node,
relationship.rel_name,
object_node,
ui=relationship.ui,
src=relationship.src,
**attrs)
def _create_atom_nodes(concept):
concept_node = ui_to_node[concept.ui]
for atom in concept.get_atoms():
atom._prefix = "DA"
atom_node = p2n.Node(f"{concept.concept_type}_ATOM",
ui=atom.ui,
name=atom.term,
src=atom.src,
src_id=atom.src_id,
is_preferred=atom.is_preferred,
**atom.attrs)
graph.create(
p2n.Relationship(concept_node, "has_synonym", atom_node))
def create_node(label, code, name, parent_code=None):
node1 = graph.nodes.match(label, code=code).first()
if node1:
print("Node: " + name + " exists!")
pass
else:
node = py2neo.Node(label, code=code, name=name)
graph.create(node)
print("Create Node: " + name)
if parent_code:
parent_node = graph.nodes.match(label, code=parent_code).first()
#print(parent_node)
rel = py2neo.Relationship(node, "属于", parent_node)
graph.create(rel)
print("Create Relationship between Node " + name + " and Node " +
parent_node['name'])
def to_neo4j(graph, neo_connection, context=None):
"""Uploads a BEL graph to Neo4J graph database using :mod:`py2neo`
:param BELGraph graph: A BEL Graph
:param neo_connection: A :mod:`py2neo` connection object. Refer to the
`py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_
for how to build this object.
:type neo_connection: :class:`py2neo.Graph`
:param str context: A disease context to allow for multiple disease models in one neo4j instance.
Each edge will be assigned an attribute :code:`pybel_context` with this value
Example Usage:
>>> import pybel, py2neo
>>> url = 'http://resource.belframework.org/belframework/1.0/knowledge/small_corpus.bel'
>>> g = pybel.from_url(url)
>>> neo_graph = py2neo.Graph("http://localhost:7474/db/data/") # use your own connection settings
>>> pybel.to_neo4j(g, neo_graph)
"""
import py2neo
tx = neo_connection.begin()
node_map = {}
for node, data in graph.nodes(data=True):
node_type = data[FUNCTION]
attrs = {k: v for k, v in data.items() if k not in {FUNCTION, NAME}}
attrs['name'] = calculate_canonical_name(graph, node)
if NAME in data:
attrs['identifier'] = data[NAME]
node_map[node] = py2neo.Node(node_type, bel=node_to_bel(data), **attrs)
tx.create(node_map[node])
for u, v, data in graph.edges(data=True):
rel_type = data[RELATION]
attrs = flatten_dict(data)
if context is not None:
attrs[PYBEL_CONTEXT_TAG] = str(context)
rel = py2neo.Relationship(node_map[u], rel_type, node_map[v], **attrs)
tx.create(rel)
tx.commit()
def export_neo4j(graph, uri, node_queue=100, edge_queue=100):
"""Export hetnet to neo4j"""
db_graph = py2neo.Graph(uri)
# Delete all existing nodes
db_graph.delete_all()
# Create uniqueness constrains and indexes
for metanode in graph.metagraph.get_nodes():
label = as_label(metanode)
if 'identifier' not in db_graph.schema.get_uniqueness_constraints(
label):
db_graph.schema.create_uniqueness_constraint(label, 'identifier')
if 'name' not in db_graph.schema.get_indexes(label):
db_graph.schema.create_index(label, 'name')
# Create nodes
creator = Creator(db_graph, node_queue)
for node in graph.get_nodes():
label = as_label(node.metanode)
data = sanitize_data(node.data)
neo_node = py2neo.Node(label,
identifier=node.identifier,
name=node.name,
**data)
creator.append(neo_node)
creator.create()
# Create edges
creator = Creator(db_graph, edge_queue)
for edge in graph.get_edges(exclude_inverts=True):
metaedge = edge.metaedge
rel_type = as_type(metaedge)
source_label = as_label(metaedge.source)
target_label = as_label(metaedge.target)
source = db_graph.find_one(source_label, 'identifier',
edge.source.identifier)
target = db_graph.find_one(target_label, 'identifier',
edge.target.identifier)
data = sanitize_data(edge.data)
neo_rel = py2neo.Relationship(source, rel_type, target, **data)
creator.append(neo_rel)
creator.create()
return db_graph
def get_tasks(self):
tasks = []
# relationships = []
for f in self.followings:
if f['hashid']:
dst_node = self.merge_node(f)
follows = py2neo.Relationship(self.src_node, 'FOLLOWS',
dst_node)
# relationships.append(follows)
self.g.create_unique(follows)
if not rh.is_user_crawled(f['domain']):
tasks.append(f['domain'].encode('utf-8'))
# if len(relationships) > 0:
# self.g.create_unique(*relationships)
return tasks
def update(self):
"""Updates all hosts' and connections' HMMs using the forward algorithm and the given evidence up to this point, then updates the database"""
# First process the newly received evidence
self.process_evidence(self.evidence)
# Apply forward algorithm
for h in self.hosts:
self.hosts[h].update()
for c in self.connections:
self.connections[c].update()
# c is connection tuple consisting of two network address + port tuples
# Extract the source and destination hosts' network address tuples
src = c[0]
dst = c[1]
# Raw address string of form 3.3.3.3:21 that will be used as a unique identifier in the database
src_raw = str(c[0][0]) + ":" + str(c[0][1])
dst_raw = str(c[1][0]) + ":" + str(c[1][1])
# Fetch the source and destination hosts' HMMs, then src_hmm.p is the probability the source host exists given the evidence up to this point
src_hmm = self.hosts[src]
dst_hmm = self.hosts[dst]
if src[0] == "8.8.8.8":
print("p = ", src_hmm.p)
# Now get the probability the connection exists
conn_p = self.connections[c].p
# Create nodes and a relationship to insert into the database
src_node = neo.Node("Host",
addr=src_raw,
ip=src[0],
port=src[1],
p=src_hmm.p[0])
dst_node = neo.Node("Host",
addr=dst_raw,
ip=dst[0],
port=dst[1],
p=dst_hmm.p[0])
connection = neo.Relationship(src_node,
"CONNECTED",
dst_node,
p=conn_p[0])
neo.Graph.merge(graph, connection, "Host", "addr")
def load_friendship(user, friend):
"""Stores a lastfm user's friend in neo4j
Args:
user (str): lastfm user name
friend (str): lastfm user name
"""
user_node = GRAPH.find_one('User',
property_key='name',
property_value=user)
friend_node = GRAPH.find_one('User',
property_key='name',
property_value=friend)
friendship = GRAPH.match_one(start_node=user_node,
rel_type='FRIENDS',
end_node=friend_node,
bidirectional=True)
if friendship is None:
friendship = py2neo.Relationship(user_node, 'FRIENDS', friend_node)
GRAPH.create(friendship)
def write_neo(nodes, edges):
if not common.settings.neo4j.getboolean("Available"):
return
print("writing graph to neo")
graph = get_graph()
graph.delete_all()
transaction = graph.begin()
count = 0
for f, t, data in tqdm(edges):
count += 1
node_from = py2neo.Node(nodes[f]["label"], **{**nodes[f], **{"id_str": f}})
node_to = py2neo.Node(nodes[t]["label"], **{**nodes[t], **{"id_str": t}})
edge = py2neo.Relationship(node_from, data["label"], node_to, **data)
transaction.merge(edge)
if count == common.settings.neo4j.getint("TransactionsBeforeCommit"):
count = 0
transaction.commit()
transaction = graph.begin()
transaction.commit()
async def create_dict(
self,
user_id: str,
dict_id: str,
words: List[str],
private: bool = False,
name: Optional[str] = None,
has_access: Optional[List[str]] = None,
) -> Dict[str, Any]:
self.logger.debug("adding new document")
tx = self.graph_db.begin()
# check that Document with the same id
dict_with_same_name = tx.graph.nodes.match(
"Dictionary",
dict_id=dict_id,
).first()
if dict_with_same_name is not None:
raise DictNameError
# create Dict
dict_node = py2neo.Node(
"Dictionary",
dict_id=dict_id,
words=words,
private=private,
name=name,
)
tx.create(dict_node)
# connect Dict with creator
author = tx.graph.nodes.match("user", user_id=user_id).first()
tx.create(py2neo.Relationship(author, "created", dict_node))
tx.commit()
def run(xconfig):
"""
build graph in Neo4j based on the Kanji dataset
"""
# check for extra options
margs = xconfig.run.modargs
# connect to Mongo
mgx = mongo(xconfig.mongo.uri)
# get all kanji from Mongo
kset = mgx.find("kanji", {})
logthis("** Kanji objects:", suffix=len(kset), loglevel=LL.INFO)
# connect to Neo4j
try:
neo = py2neo.Graph(xconfig.neo4j.uri)
ncount = neo.cypher.execute('MATCH (n) RETURN count(*)')
except Exception as e:
logexc(e, "Failed to connect to Neo4j dataset")
failwith(ER.PROCFAIL, "Unable to continue. Aborting.")
# if 'clear' is passed as an extra parg, then drop all existing nodes/rels
if 'clear' in margs:
logthis("Deleting existing data...", loglevel=LL.INFO)
neo.cypher.execute("MATCH (n) DETACH DELETE n")
# create node constraints
logthis("Creating constraints...", loglevel=LL.VERBOSE)
neo.cypher.execute(
"CREATE CONSTRAINT ON (k:Kanji) ASSERT k.kanji IS UNIQUE")
neo.cypher.execute(
"CREATE CONSTRAINT ON (r:Radical) ASSERT r.radical IS UNIQUE")
neo.cypher.execute(
"CREATE CONSTRAINT ON (s:Sense) ASSERT s.sense IS UNIQUE")
neo.cypher.execute(
"CREATE CONSTRAINT ON (r:Reading) ASSERT r.reading IS UNIQUE")
neo.cypher.execute("CREATE CONSTRAINT ON (g:Joyo) ASSERT g.joyo IS UNIQUE")
neo.cypher.execute("CREATE CONSTRAINT ON (g:Jlpt) ASSERT g.jlpt IS UNIQUE")
neo.cypher.execute("CREATE CONSTRAINT ON (k:Skip) ASSERT k.skip IS UNIQUE")
# Build nodes & relationships
logthis("** Building graph...", loglevel=LL.INFO)
for kk, tk in kset.iteritems():
logthis(">>>------[ %5d ] Kanji node <%s> -----" % (kk, tk['kanji']),
loglevel=LL.DEBUG)
# Kanji
try:
freq = int(tk['freq'])
except:
freq = 0
knode = py2neo.Node("Kanji",
kanji=tk['kanji'],
ucs=tk['_id'],
freq=freq)
# Radicals
xnodes = []
xrels = []
if tk.has_key('xrad') and len(tk['xrad']) > 0:
for tr, tv in tk['xrad'].iteritems():
# check if a radical exists in db.radical
rrad = mgx.findOne("radical", {"radical": tr})
xrad = {}
if rrad:
xrad = {
"rad_id": rrad['_id'],
"alt": rrad['alt'],
"radname": rrad['radname']['ja'],
"radname_en": rrad['radname']['en']
}
else:
rrad = mgx.findOne("kanji", {"kanji": tr})
if rrad:
# Created Kanji-Kanji relationship
xrad = False
try:
freq = int(rrad['freq'])
except:
freq = 0
xnodes.append(
py2neo.Node("Kanji",
kanji=rrad['kanji'],
ucs=rrad['_id'],
freq=freq))
xrels.append(
py2neo.Relationship(knode,
"CONTAINS",
xnodes[-1],
position=tv.get(
'position', None)))
else:
xrad = {"non_standard": True}
if xrad:
xnodes.append(py2neo.Node("Radical", radical=tr, **xrad))
xrels.append(
py2neo.Relationship(knode,
"CONTAINS",
xnodes[-1],
position=tv.get('position', None)))
elif tk.has_key('krad'):
for tr in tk['krad']:
# check if a radical exists in db.radical
rrad = mgx.findOne("radical", {"radical": tr})
xrad = {}
if rrad:
xrad = {
"rad_id": rrad['_id'],
"alt": rrad['alt'],
"radname": rrad['radname']['ja'],
"radname_en": rrad['radname']['en']
}
else:
rrad = mgx.findOne("kanji", {"kanji": tr})
if rrad:
# Created Kanji-Kanji relationship
xrad = False
try:
freq = int(rrad['freq'])
except:
freq = 0
xnodes.append(
py2neo.Node("Kanji",
kanji=rrad['kanji'],
ucs=rrad['_id'],
freq=freq))
xrels.append(
py2neo.Relationship(knode, "CONTAINS", xnodes[-1]))
else:
xrad = {"non_standard": True}
if xrad:
xnodes.append(py2neo.Node("Radical", radical=tr, **xrad))
xrels.append(
py2neo.Relationship(knode, "CONTAINS", xnodes[-1]))
# Senses
if tk.has_key('meaning') and tk['meaning'].get('en'):
for ts in tk['meaning']['en']:
xnodes.append(py2neo.Node("Sense", sense=ts, lang="en"))
xrels.append(py2neo.Relationship(knode, "MEANS", xnodes[-1]))
# Readings (on-yomi, kun-yomi, nanori)
if tk.has_key('reading'):
if tk['reading'].has_key('ja_on'):
for tr in tk['reading']['ja_on']:
xnodes.append(py2neo.Node("Reading", reading=tr))
xrels.append(
py2neo.Relationship(knode,
"READS",
xnodes[-1],
yomi="on"))
if tk['reading'].has_key('ja_kun'):
for tr in tk['reading']['ja_kun']:
xnodes.append(py2neo.Node("Reading", reading=tr))
xrels.append(
py2neo.Relationship(knode,
"READS",
xnodes[-1],
yomi="kun"))
if tk['reading'].has_key('nanori'):
for tr in tk['reading']['nanori']:
xnodes.append(py2neo.Node("Reading", reading=tr))
xrels.append(
py2neo.Relationship(knode,
"READS",
xnodes[-1],
yomi="nanori"))
# Joyo
if tk.has_key('grade') and tk.has_key('jindex'):
xnodes.append(py2neo.Node("Joyo", joyo=int(tk['grade'])))
xrels.append(
py2neo.Relationship(xnodes[-1],
"SUBSET",
knode,
jindex=tk['jindex']))
# JLPT
if tk.has_key('jlpt') and isinstance(tk['jlpt'], int):
xnodes.append(py2neo.Node("Jlpt", jlpt=int(tk['jlpt'])))
xrels.append(py2neo.Relationship(xnodes[-1], "SUBSET", knode))
# SKIP
if tk.has_key('qcode') and tk['qcode'].has_key('skip'):
xnodes.append(py2neo.Node("Skip", skip=tk['qcode']['skip']))
xrels.append(py2neo.Relationship(knode, "WRITTEN", xnodes[-1]))
# Create Kanji node
try:
neo.create(knode)
except Exception as e:
logexc(e, u'Failed to create Kanji node')
# Create nodes
for tnode in xnodes:
try:
neo.create(tnode)
except Exception as e:
logexc(e, u'Failed to create aux node')
# Build relations
for trel in xrels:
# Check if Nodes are bound
sn = trel.start_node
en = trel.end_node
# if start node is not bound, then attempt a lookup
if not sn.bound:
nlab = list(sn.labels)[0]
nsn = neo.find_one(nlab, nlab.lower(), sn[nlab.lower()])
if nsn:
logthis(">>> Xref OK: %s '%s'" % (nlab, sn[nlab.lower()]),
loglevel=LL.DEBUG)
sn = nsn
# if end node is not bound, then attempt a lookup
if not en.bound:
elab = list(en.labels)[0]
nen = neo.find_one(elab, elab.lower(), en[elab.lower()])
if nen:
logthis(">>> Xref OK: %s '%s'" % (elab, en[elab.lower()]),
loglevel=LL.DEBUG)
en = nen
# Rebuild relationship
rrel = py2neo.Relationship(sn, trel.type, en, **trel.properties)
try:
neo.create_unique(rrel)
except Exception as e:
logexc(e, "Failed to build relationship")
def to_neo4j(graph, neo_connection, use_tqdm=False):
"""Upload a BEL graph to a Neo4j graph database using :mod:`py2neo`.
:param pybel.BELGraph graph: A BEL Graph
:param neo_connection: A :mod:`py2neo` connection object. Refer to the
`py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_ for how to build this object.
:type neo_connection: str or py2neo.Graph
Example Usage:
>>> import py2neo
>>> import pybel
>>> from pybel.examples import sialic_acid_graph
>>> neo_graph = py2neo.Graph("http://localhost:7474/db/data/") # use your own connection settings
>>> pybel.to_neo4j(sialic_acid_graph, neo_graph)
"""
import py2neo
if isinstance(neo_connection, str):
neo_connection = py2neo.Graph(neo_connection)
tx = neo_connection.begin()
node_map = {}
nodes = list(graph)
if use_tqdm:
nodes = tqdm(nodes, desc='nodes')
for node in nodes:
if NAMESPACE not in node or VARIANTS in node or MEMBERS in node or FUSION in node:
attrs = {'name': node.as_bel()}
else:
attrs = {'namespace': node.namespace}
if node.name and node.identifier:
attrs['name'] = node.name
attrs['identifier'] = node.identifier
elif node.identifier and not node.name:
attrs['name'] = node.identifier
elif node.name and not node.identifier:
attrs['name'] = node.name
node_map[node] = py2neo.Node(node.function, **attrs)
tx.create(node_map[node])
edges = graph.edges(keys=True, data=True)
if use_tqdm:
edges = tqdm(edges, desc='edges')
for u, v, key, node in edges:
rel_type = node[RELATION]
d = node.copy()
del d[RELATION]
attrs = {}
annotations = d.pop(ANNOTATIONS, None)
if annotations:
for annotation, values in annotations.items():
attrs[annotation] = list(values)
citation = d.pop(CITATION, None)
if citation:
attrs[CITATION] = '{}:{}'.format(citation[CITATION_TYPE], citation[CITATION_REFERENCE])
if EVIDENCE in d:
attrs[EVIDENCE] = d[EVIDENCE]
for side in (SUBJECT, OBJECT):
side_data = d.get(side)
if side_data:
attrs.update(flatten_dict(side_data, parent_key=side))
rel = py2neo.Relationship(node_map[u], rel_type, node_map[v], key=key, **attrs)
tx.create(rel)
tx.commit()