def create_followers(node):
"""
为参数结点在数据库中生成follower
:param node: label为developer的结点
:return:
"""
info = {}
try:
info = requests.get(node["followers_url"], headers=headers).json()
except KeyError:
print("Node error")
exit(-1)
if info:
followers = [member["url"] for member in info]
for follower in followers:
follower_info = requests.get(follower, headers=headers).json()
properties, relationships = get_developer_info(follower_info)
temp = properties.copy()
temp.update(relationships)
follower_node = py2neo.Node("DEVELOPER", **temp)
rel1 = py2neo.Relationship(node, "IS_FOLLOWED_BY", follower_node)
rel2 = py2neo.Relationship(follower_node, "FOLLOWS", node)
tx.merge(follower_node, "DEVELOPER", "id")
tx.create(rel1)
tx.create(rel2)
def create_tweet(
tweet_id: str,
user_id: str,
node_number: int,
flagged: bool,
) -> bool:
"""
takes relevant tweet details and inserts it
Args:
tweet_id (string), tweet's id for reference
user_id (string), user's id for reference
node_number (int), number indicating chronlogical order of user's tweets
flagged (boolean), indicating whether the tweet got flagged or not.
Returns:
boolean, confirming whether Tweet got created or not.
"""
tweet = py2neo.Node(TWEET_NODE_LABEL,
tweet_id=tweet_id,
user_id=user_id,
node_number=node_number,
flagged=flagged)
graph.create(tweet)
return graph.exists(tweet)
def make_systems_in_graph(settings, user_id, system_data):
graph = social_graph(settings)
system_owner = graph.find_one("User", "sql_id", user_id)
if system_owner is None:
print("user is not in graph yet (never logged in), skipping")
return
for entry in system_data:
print(entry)
try:
systemNode = py2neo.Node(
"System",
system_id=entry['id'],
system_uid=entry['system_uid'],
name=entry['name'],
description=entry['name'],
location_lat=to_float(entry['location_lat']),
location_lng=to_float(entry['location_lng']),
status=entry['status'],
creation_time=timestamp(),
modified_time=timestamp())
graph.create(systemNode)
relationship = py2neo.Relationship(system_owner, "SYS_ADMIN",
systemNode)
graph.create(relationship)
except Exception as e:
tb.print_exc()
raise e
def processItem(item):
neo4jGraph, topologicGraph, labelKey, relationshipKey, bidirectional, deleteAll, tolerance, run = item
if not (run):
return None
import time
gmt = time.gmtime()
timestamp = str(gmt.tm_zone) + "_" + str(gmt.tm_year) + "_" + str(
gmt.tm_mon) + "_" + str(gmt.tm_wday) + "_" + str(
gmt.tm_hour) + "_" + str(gmt.tm_min) + "_" + str(gmt.tm_sec)
vertices = []
_ = topologicGraph.Vertices(vertices)
edges = []
_ = topologicGraph.Edges(edges)
notUsed = []
tx = neo4jGraph.begin()
nodes = []
for i in range(len(vertices)):
vDict = vertices[i].GetDictionary()
keys, values = getKeysAndValues(vDict)
keys.append("x")
keys.append("y")
keys.append("z")
keys.append("timestamp")
keys.append("location")
values.append(vertices[i].X())
values.append(vertices[i].Y())
values.append(vertices[i].Z())
values.append(timestamp)
values.append(
sp.CartesianPoint(
[vertices[i].X(), vertices[i].Y(), vertices[i].Z()]))
zip_iterator = zip(keys, values)
pydict = dict(zip_iterator)
if (labelKey == 'None') or (not (labelKey)):
nodeName = "TopologicGraphVertex"
else:
nodeName = str(getValueAtKey(vDict, labelKey))
n = py2neo.Node(nodeName, **pydict)
tx.create(n)
nodes.append(n)
for i in range(len(edges)):
e = edges[i]
sv = e.StartVertex()
ev = e.EndVertex()
sn = nodes[vertexIndex(sv, vertices, tolerance)]
en = nodes[vertexIndex(ev, vertices, tolerance)]
ed = e.GetDictionary()
relationshipType = getValueAtKey(ed, relationshipKey)
if not (relationshipType):
relationshipType = "Connected To"
snen = py2neo.Relationship(sn, relationshipType, en)
tx.create(snen)
if bidirectional:
snen = py2neo.Relationship(en, relationshipType, sn)
tx.create(snen)
if deleteAll:
neo4jGraph.delete_all()
neo4jGraph.commit(tx)
return neo4jGraph
def maker(icls, attr, dimension):
individual = tools.initRepeat(icls, attr, dimension)
node = py2neo.Node("I", str(SEED), gene=individual, gen=0, fitness=None)
neoNodes.append(node)
neodb.graph.merge(node)
individual.neoNodeIndex = len(neoNodes) - 1
print(individual.neoNodeIndex)
return individual
def create_node(label, node_ls, kg):
for n in node_ls:
node = py2neo.Node(label, name=n)
print(node)
try:
kg.create(node)
except:
print("建立%s节点时报错" % label)
def export_neo4j(graph, uri, node_queue=200, edge_queue=5, show_progress=False):
"""Export hetnet to neo4j"""
if show_progress:
from tqdm import tqdm
py2neo, _ = import_py2neo()
if isinstance(uri, py2neo.Graph):
db_graph = uri
else:
db_graph = py2neo.Graph(uri)
# Delete all existing nodes
db_graph.delete_all()
# Create uniqueness constrains and indexes
for metanode in graph.metagraph.get_nodes():
label = metanode.neo4j_label
if "identifier" not in db_graph.schema.get_uniqueness_constraints(label):
db_graph.schema.create_uniqueness_constraint(label, "identifier")
if "name" not in db_graph.schema.get_indexes(label):
db_graph.schema.create_index(label, "name")
# Create nodes
creator = Creator(db_graph, node_queue)
queue = graph.get_nodes()
if show_progress:
queue = tqdm(queue, total=graph.n_nodes, desc="Importing nodes")
for node in queue:
label = node.metanode.neo4j_label
data = sanitize_data(node.data)
neo_node = py2neo.Node(
label, identifier=node.identifier, name=node.name, **data
)
creator.append(neo_node)
creator.create()
# Create edges
creator = Creator(db_graph, edge_queue)
queue = graph.get_edges(exclude_inverts=True)
if show_progress:
queue = tqdm(queue, total=graph.n_edges, desc="Importing edges")
for edge in queue:
metaedge = edge.metaedge
rel_type = metaedge.neo4j_rel_type
source_label = metaedge.source.neo4j_label
target_label = metaedge.target.neo4j_label
source = db_graph.find_one(source_label, "identifier", edge.source.identifier)
target = db_graph.find_one(target_label, "identifier", edge.target.identifier)
data = sanitize_data(edge.data)
neo_rel = py2neo.Relationship(source, rel_type, target, **data)
creator.append(neo_rel)
creator.create()
return db_graph
def create_log_node(dataloader_name, image):
n = py2neo.Node("LoadingLog")
n["loader"] = dataloader_name
n["dockerhub_image_name"] = image.tags[0].split(":")[0]
n["dockerhub_image_tag"] = image.tags[0].split(":")[1]
n["dockerhub_image_hash"] = image.id
n["loading_finished_at"] = str(datetime.datetime.now(tz=None))
tx = get_graph().begin()
tx.create(n)
tx.commit()
def _get_or_create_node(self, node_type, **kwargs):
name = kwargs['name']
node_id = self.node_info_to_id_cache.get((node_type, name), None)
if node_id is None:
node = py2neo.Node(node_type.name, **kwargs)
self.graph_db.create(node)
node_id = self._get_node_id(node)
self.node_info_to_id_cache[(node_type, name)] = node_id
return node
return self.graph_db.node(node_id)
def _convert_concept_to_node(concept):
concept._prefix = "DC"
attrs = {
atr.atr_name: atr.atr_value
for atr in concept.get_attributes()
}
concept_node = p2n.Node(concept.concept_type,
ui=concept.ui,
name=concept.preferred_atom.term,
**attrs)
return concept_node
def update(self):
"""Updates all hosts' and connections' HMMs using the forward algorithm and the given evidence up to this point, then updates the database"""
# First process the newly received evidence
self.process_evidence(self.evidence)
# Apply forward algorithm
for h in self.hosts:
self.hosts[h].update()
for c in self.connections:
self.connections[c].update()
# c is connection tuple consisting of two network address + port tuples
# Extract the source and destination hosts' network address tuples
src = c[0]
dst = c[1]
# Raw address string of form 3.3.3.3:21 that will be used as a unique identifier in the database
src_raw = str(c[0][0]) + ":" + str(c[0][1])
dst_raw = str(c[1][0]) + ":" + str(c[1][1])
# Fetch the source and destination hosts' HMMs, then src_hmm.p is the probability the source host exists given the evidence up to this point
src_hmm = self.hosts[src]
dst_hmm = self.hosts[dst]
if src[0] == "8.8.8.8":
print("p = ", src_hmm.p)
# Now get the probability the connection exists
conn_p = self.connections[c].p
# Create nodes and a relationship to insert into the database
src_node = neo.Node("Host",
addr=src_raw,
ip=src[0],
port=src[1],
p=src_hmm.p[0])
dst_node = neo.Node("Host",
addr=dst_raw,
ip=dst[0],
port=dst[1],
p=dst_hmm.p[0])
connection = neo.Relationship(src_node,
"CONNECTED",
dst_node,
p=conn_p[0])
neo.Graph.merge(graph, connection, "Host", "addr")
def load_user_info(user):
"""Stores a lastfm user in neo4j
Args:
user (dict): lastfm user
"""
user_node = GRAPH.find_one('User',
property_key='name',
property_value=user['name'])
if user_node is None:
user_node = py2neo.Node('User', **user)
GRAPH.create(user_node)
def processItem(item):
import time
gmt = time.gmtime()
timestamp = str(gmt.tm_zone)+"_"+str(gmt.tm_year)+"_"+str(gmt.tm_mon)+"_"+str(gmt.tm_wday)+"_"+str(gmt.tm_hour)+"_"+str(gmt.tm_min)+"_"+str(gmt.tm_sec)
neo4jGraph, topologicGraph, categoryKey, tolerance = item
vertices = []
_ = topologicGraph.Vertices(vertices)
edges = []
_ = topologicGraph.Edges(edges)
notUsed = []
tx = neo4jGraph.begin()
nodes = []
for i in range(len(vertices)):
vDict = vertices[i].GetDictionary()
keys, values = getKeysAndValues(vDict)
keys.append("x")
keys.append("y")
keys.append("z")
keys.append("timestamp")
keys.append("location")
values.append(vertices[i].X())
values.append(vertices[i].Y())
values.append(vertices[i].Z())
values.append(timestamp)
values.append(sp.CartesianPoint([vertices[i].X(),vertices[i].Y(),vertices[i].Z()]))
zip_iterator = zip(keys, values)
pydict = dict(zip_iterator)
if categoryKey == 'None':
nodeName = "TopologicGraphVertex"
else:
nodeName = str(values[keys.index(categoryKey)])
n = py2neo.Node(nodeName, **pydict)
neo4jGraph.cypher.execute("CREATE INDEX FOR (n:%s) on (n.name)" %
n.nodelabel)
#try:
#neo4jGraph.cypher.execute("CREATE INDEX FOR (n:%s) on (n.name)" %
#n.nodelabel)
#except:
#pass
tx.create(n)
nodes.append(n)
for i in range(len(edges)):
e = edges[i]
sv = e.StartVertex()
ev = e.EndVertex()
sn = nodes[vertexIndex(sv, vertices, tolerance)]
en = nodes[vertexIndex(ev, vertices, tolerance)]
snen = py2neo.Relationship(sn, "CONNECTEDTO", en)
tx.create(snen)
snen = py2neo.Relationship(en, "CONNECTEDTO", sn)
tx.create(snen)
neo4jGraph.commit(tx)
return neo4jGraph
def addNode(tx, tabNode, pcInfo):
for node in tabNode:
if (node["ipAddress"] == pcInfo[0]):
return node, 0
label = "PublicIp"
if (ipaddress.ip_address(pcInfo[0]).is_private):
label = "PrivateIp"
newNode = py2neo.Node(label, ipAddress=pcInfo[0], macAddress=pcInfo[1])
tabNode.append(newNode)
tx.create(newNode)
return newNode, 1
def write_neo(nodes, edges):
if not common.settings.neo4j.getboolean("Available"):
return
print("writing graph to neo")
graph = get_graph()
graph.delete_all()
transaction = graph.begin()
count = 0
for f, t, data in tqdm(edges):
count += 1
node_from = py2neo.Node(nodes[f]["label"], **{**nodes[f], **{"id_str": f}})
node_to = py2neo.Node(nodes[t]["label"], **{**nodes[t], **{"id_str": t}})
edge = py2neo.Relationship(node_from, data["label"], node_to, **data)
transaction.merge(edge)
if count == common.settings.neo4j.getint("TransactionsBeforeCommit"):
count = 0
transaction.commit()
transaction = graph.begin()
transaction.commit()
def getNeo4j(self):
"""Get a neo4j object"""
# define properties
properties = {}
for key in self.properties:
properties[key] = getattr(self, key)
node = py2neo.Node(self.label, **properties)
return node
def create(self, name, properties={}, collection=BaseGraphDB.DEFAULT_COLLECTION):
log.log.debug(
'Creating %s, name %s with properties %s',
collection.name,
name,
properties
)
properties = deepcopy(properties)
properties['name'] = name
n = py2neo.Node(collection.name, **properties)
return self._r.create(n)[0]
def _create_atom_nodes(concept):
concept_node = ui_to_node[concept.ui]
for atom in concept.get_atoms():
atom._prefix = "DA"
atom_node = p2n.Node(f"{concept.concept_type}_ATOM",
ui=atom.ui,
name=atom.term,
src=atom.src,
src_id=atom.src_id,
is_preferred=atom.is_preferred,
**atom.attrs)
graph.create(
p2n.Relationship(concept_node, "has_synonym", atom_node))
def load_artist(artist):
"""Stores a lastfm artist in neo4j
Args:
artist (dict): lastfm artist
"""
artist_node = GRAPH.find_one('Artist',
property_key='url',
property_value=artist['url'])
if artist_node is None:
artist_node = py2neo.Node('Artist', **artist)
GRAPH.create(artist_node)
def cxSet(ind1, ind2):
"""Apply a crossover operation on input sets. The first child is the
intersection of the two sets, the second child is the difference of the
two sets.
"""
p1 = neoNodes[ind1.neoNodeIndex]
p2 = neoNodes[ind2.neoNodeIndex]
temp = set(ind1) # Used in order to keep type
ind1 &= ind2 # Intersection (inplace)
ind2 ^= temp # Symmetric Difference (inplace)
node1 = py2neo.Node(gene=ind1, gen=None)
node2 = py2neo.Node(gene=ind2, gen=None)
r1 = peaviz.ParentOf(p1, node1)
r2 = peaviz.ParentOf(p2, node2)
neoNodes.append(node1)
neoNodes.append(node2)
ind2.neoNodeIndex = len(neoNodes) - 1
ind1.neoNodeIndex = ind2.neoNodeIndex - 1
neodb.graph.merge(node1 | node2 | r1 | r2)
return ind1, ind2
def create_node(label, code, name, parent_code=None):
node1 = graph.nodes.match(label, code=code).first()
if node1:
print("Node: " + name + " exists!")
pass
else:
node = py2neo.Node(label, code=code, name=name)
graph.create(node)
print("Create Node: " + name)
if parent_code:
parent_node = graph.nodes.match(label, code=parent_code).first()
#print(parent_node)
rel = py2neo.Relationship(node, "属于", parent_node)
graph.create(rel)
print("Create Relationship between Node " + name + " and Node " +
parent_node['name'])
def __init__(self, cfg: SimpleNamespace, storage_dir: Path,
auth_module: BaseAuth):
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.getLogger("paperback").level)
self.logger.info("initializing papertext_docs module")
self.logger.debug("using storage dir %s", storage_dir)
self.logger.debug("using config %s", cfg)
self.storage_dir: Path = storage_dir
self.cfg: SimpleNamespace = cfg
# TODO: add check for configuration,
# i.e. that hash.algo lib and token.curve lib are present
self.auth_module = auth_module
self.docs_backup_folder = self.storage_dir / "docs.bak"
self.docs_backup_folder.mkdir(parents=True, exist_ok=True)
self.logger.debug("connecting to neo4j database")
self.graph_db = py2neo.Graph(
scheme=self.cfg.db.scheme,
user=self.cfg.db.username,
password=self.cfg.db.password,
host=self.cfg.db.host,
port=self.cfg.db.port,
)
self.logger.debug("connected to neo4j database")
self.logger.debug("creating default corpus")
self.root_corp = self.graph_db.nodes.match("corp",
corp_id="root").first()
if self.root_corp is None:
tx = self.graph_db.begin()
self.root_corp = py2neo.Node("corp", corp_id="root")
tx.create(self.root_corp)
tx.commit()
self.logger.debug("created default root corpus")
else:
self.logger.debug("using already existing root corpus")
self.logger.debug("syncing with auth module")
self.sync_modules_on_startup()
self.logger.debug("synced with auth module")
self.logger.debug("loading analyzers")
self.analyzers = self.get_analyzers(cfg.analyzers)
self.logger.debug("loaded analyzers")
def parse_yearpage (self, response) :
"""
Process a single year-based recipe directory page.
The flow obtains a list of year-recipe URLs and then crawls each URL.
Parameters:
self: Object instance
response: Selector object provided by invoking parent method.
"""
# Log iteration
logger.info(f'parse_yearpage for {response.request.url}')
# Increment number of directory pages traveled.
self.n_directory_pages = self.n_directory_pages + 1
# Do not process further if the max counts have been passed.
if self.n_directory_pages > MAX_DIRECTORY_PAGES :
logger.info(f'Passed threshold of directory pages. ({self.n_directory_pages})')
return
# Create a recipe directory node
logger.info(f'Creating new recipe page node in {self.gdb.database.name}')
tx = py2neo.begin()
rdir = py2neo.Node("RecipeDirectory", url=response.request.url)
tx.py2neo.create(rdir)
tx.commit()
if not food_db.exists(rdir) :
logger.error(f'Recipe directory {response.request.url} was not created. :()')
css_str = ' div#sitemapItems'
xpath_ul_str = './/div/h1[contains(text(), "Recipes")]/following-sibling::ul'
xpath_recipelink_str = './/li/a/@href'
xpath_nextpage_str = './/div[@class = "paginate"]/a[@title="Next page"]/@href'
# Obtain URLs for all recipes on a page
recipeitems = response.css(css_str).xpath(xpath_ul_str).xpath(xpath_recipelink_str)
logger.info(recipeitems.getall())
for recipe_link in recipeitems:
pass
# Is there another page for this year?
nextpageitems = response.css(css_str).xpath(xpath_nextpage_str)
logger.info (f'Next Page: { nextpageitems.get() }')
if len(nextpageitems) > 0 :
nextpage_link = nextpageitems.get()
yield response.follow(url = nextpage_link, callback = self.parse_yearpage)
def to_neo4j(graph, neo_connection, context=None):
"""Uploads a BEL graph to Neo4J graph database using :mod:`py2neo`
:param BELGraph graph: A BEL Graph
:param neo_connection: A :mod:`py2neo` connection object. Refer to the
`py2neo documentation <http://py2neo.org/v3/database.html#the-graph>`_
for how to build this object.
:type neo_connection: :class:`py2neo.Graph`
:param str context: A disease context to allow for multiple disease models in one neo4j instance.
Each edge will be assigned an attribute :code:`pybel_context` with this value
Example Usage:
>>> import pybel, py2neo
>>> url = 'http://resource.belframework.org/belframework/1.0/knowledge/small_corpus.bel'
>>> g = pybel.from_url(url)
>>> neo_graph = py2neo.Graph("http://localhost:7474/db/data/") # use your own connection settings
>>> pybel.to_neo4j(g, neo_graph)
"""
import py2neo
tx = neo_connection.begin()
node_map = {}
for node, data in graph.nodes(data=True):
node_type = data[FUNCTION]
attrs = {k: v for k, v in data.items() if k not in {FUNCTION, NAME}}
attrs['name'] = calculate_canonical_name(graph, node)
if NAME in data:
attrs['identifier'] = data[NAME]
node_map[node] = py2neo.Node(node_type, bel=node_to_bel(data), **attrs)
tx.create(node_map[node])
for u, v, data in graph.edges(data=True):
rel_type = data[RELATION]
attrs = flatten_dict(data)
if context is not None:
attrs[PYBEL_CONTEXT_TAG] = str(context)
rel = py2neo.Relationship(node_map[u], rel_type, node_map[v], **attrs)
tx.create(rel)
tx.commit()
def export_neo4j(graph, uri, node_queue=100, edge_queue=100):
"""Export hetnet to neo4j"""
db_graph = py2neo.Graph(uri)
# Delete all existing nodes
db_graph.delete_all()
# Create uniqueness constrains and indexes
for metanode in graph.metagraph.get_nodes():
label = as_label(metanode)
if 'identifier' not in db_graph.schema.get_uniqueness_constraints(
label):
db_graph.schema.create_uniqueness_constraint(label, 'identifier')
if 'name' not in db_graph.schema.get_indexes(label):
db_graph.schema.create_index(label, 'name')
# Create nodes
creator = Creator(db_graph, node_queue)
for node in graph.get_nodes():
label = as_label(node.metanode)
data = sanitize_data(node.data)
neo_node = py2neo.Node(label,
identifier=node.identifier,
name=node.name,
**data)
creator.append(neo_node)
creator.create()
# Create edges
creator = Creator(db_graph, edge_queue)
for edge in graph.get_edges(exclude_inverts=True):
metaedge = edge.metaedge
rel_type = as_type(metaedge)
source_label = as_label(metaedge.source)
target_label = as_label(metaedge.target)
source = db_graph.find_one(source_label, 'identifier',
edge.source.identifier)
target = db_graph.find_one(target_label, 'identifier',
edge.target.identifier)
data = sanitize_data(edge.data)
neo_rel = py2neo.Relationship(source, rel_type, target, **data)
creator.append(neo_rel)
creator.create()
return db_graph
def create(self, path, mode, fi=None):
''' 创建文件时 入Neo4j库 用文件大小，文件后缀名，文件所有者分组
给文件添加标签 '''
full_path = self._full_path(path)
fd = os.open(full_path, os.O_RDWR | os.O_CREAT, mode)
if GBFS_graph.find_one(label=full_path) == None:
return fd
info = os.fstat(fd)
name_tuple = os.path.splitext(full_path)
a = py2neo.Node(name_tuple[1],
uid=info.st_uid,
gid=info.st_gid,
atime=info.st_atime,
mtime=info.st_mtime,
fd=fd,
name=full_path)
GBFS_graph.create(a)
return fd
def addNode(self, node):
"""
Add new node to the DB.
:param node: node to be added to the DB
:type node: dict
:return: success of addition
:rtype: Boolean
"""
node_db = list(
self.graph_db.find(node['label'], 'node_id', node['node_id']))
if len(node_db) > 0:
log.debug("node %s exists in the DB" % node['node_id'])
return False
node_new = py2neo.Node(node['label'], node_id=node['node_id'])
for key, value in node.items():
node_new.properties[key] = value
self.graph_db.create(node_new)
return True
def create_user(user_id: str, user_name: str) -> bool:
"""
takes user's unique id and name as inputs
to insert as a node in the neo4j db
note: it will create a new node, even if
the user_id and name combo already exists.
Args:
user_id (string), user's id for reference
user_name (string), user's name for reference
Returns:
boolean, confirming whether newly onboarded User got created or not.
"""
user = py2neo.Node(USER_NODE_LABEL, user_id=user_id, name=user_name)
graph.create(user)
return graph.exists(user)
async def create_dict(
self,
user_id: str,
dict_id: str,
words: List[str],
private: bool = False,
name: Optional[str] = None,
has_access: Optional[List[str]] = None,
) -> Dict[str, Any]:
self.logger.debug("adding new document")
tx = self.graph_db.begin()
# check that Document with the same id
dict_with_same_name = tx.graph.nodes.match(
"Dictionary",
dict_id=dict_id,
).first()
if dict_with_same_name is not None:
raise DictNameError
# create Dict
dict_node = py2neo.Node(
"Dictionary",
dict_id=dict_id,
words=words,
private=private,
name=name,
)
tx.create(dict_node)
# connect Dict with creator
author = tx.graph.nodes.match("user", user_id=user_id).first()
tx.create(py2neo.Relationship(author, "created", dict_node))
tx.commit()
def run(xconfig):
"""
build graph in Neo4j based on the Kanji dataset
"""
# check for extra options
margs = xconfig.run.modargs
# connect to Mongo
mgx = mongo(xconfig.mongo.uri)
# get all kanji from Mongo
kset = mgx.find("kanji", {})
logthis("** Kanji objects:", suffix=len(kset), loglevel=LL.INFO)
# connect to Neo4j
try:
neo = py2neo.Graph(xconfig.neo4j.uri)
ncount = neo.cypher.execute('MATCH (n) RETURN count(*)')
except Exception as e:
logexc(e, "Failed to connect to Neo4j dataset")
failwith(ER.PROCFAIL, "Unable to continue. Aborting.")
# if 'clear' is passed as an extra parg, then drop all existing nodes/rels
if 'clear' in margs:
logthis("Deleting existing data...", loglevel=LL.INFO)
neo.cypher.execute("MATCH (n) DETACH DELETE n")
# create node constraints
logthis("Creating constraints...", loglevel=LL.VERBOSE)
neo.cypher.execute(
"CREATE CONSTRAINT ON (k:Kanji) ASSERT k.kanji IS UNIQUE")
neo.cypher.execute(
"CREATE CONSTRAINT ON (r:Radical) ASSERT r.radical IS UNIQUE")
neo.cypher.execute(
"CREATE CONSTRAINT ON (s:Sense) ASSERT s.sense IS UNIQUE")
neo.cypher.execute(
"CREATE CONSTRAINT ON (r:Reading) ASSERT r.reading IS UNIQUE")
neo.cypher.execute("CREATE CONSTRAINT ON (g:Joyo) ASSERT g.joyo IS UNIQUE")
neo.cypher.execute("CREATE CONSTRAINT ON (g:Jlpt) ASSERT g.jlpt IS UNIQUE")
neo.cypher.execute("CREATE CONSTRAINT ON (k:Skip) ASSERT k.skip IS UNIQUE")
# Build nodes & relationships
logthis("** Building graph...", loglevel=LL.INFO)
for kk, tk in kset.iteritems():
logthis(">>>------[ %5d ] Kanji node <%s> -----" % (kk, tk['kanji']),
loglevel=LL.DEBUG)
# Kanji
try:
freq = int(tk['freq'])
except:
freq = 0
knode = py2neo.Node("Kanji",
kanji=tk['kanji'],
ucs=tk['_id'],
freq=freq)
# Radicals
xnodes = []
xrels = []
if tk.has_key('xrad') and len(tk['xrad']) > 0:
for tr, tv in tk['xrad'].iteritems():
# check if a radical exists in db.radical
rrad = mgx.findOne("radical", {"radical": tr})
xrad = {}
if rrad:
xrad = {
"rad_id": rrad['_id'],
"alt": rrad['alt'],
"radname": rrad['radname']['ja'],
"radname_en": rrad['radname']['en']
}
else:
rrad = mgx.findOne("kanji", {"kanji": tr})
if rrad:
# Created Kanji-Kanji relationship
xrad = False
try:
freq = int(rrad['freq'])
except:
freq = 0
xnodes.append(
py2neo.Node("Kanji",
kanji=rrad['kanji'],
ucs=rrad['_id'],
freq=freq))
xrels.append(
py2neo.Relationship(knode,
"CONTAINS",
xnodes[-1],
position=tv.get(
'position', None)))
else:
xrad = {"non_standard": True}
if xrad:
xnodes.append(py2neo.Node("Radical", radical=tr, **xrad))
xrels.append(
py2neo.Relationship(knode,
"CONTAINS",
xnodes[-1],
position=tv.get('position', None)))
elif tk.has_key('krad'):
for tr in tk['krad']:
# check if a radical exists in db.radical
rrad = mgx.findOne("radical", {"radical": tr})
xrad = {}
if rrad:
xrad = {
"rad_id": rrad['_id'],
"alt": rrad['alt'],
"radname": rrad['radname']['ja'],
"radname_en": rrad['radname']['en']
}
else:
rrad = mgx.findOne("kanji", {"kanji": tr})
if rrad:
# Created Kanji-Kanji relationship
xrad = False
try:
freq = int(rrad['freq'])
except:
freq = 0
xnodes.append(
py2neo.Node("Kanji",
kanji=rrad['kanji'],
ucs=rrad['_id'],
freq=freq))
xrels.append(
py2neo.Relationship(knode, "CONTAINS", xnodes[-1]))
else:
xrad = {"non_standard": True}
if xrad:
xnodes.append(py2neo.Node("Radical", radical=tr, **xrad))
xrels.append(
py2neo.Relationship(knode, "CONTAINS", xnodes[-1]))
# Senses
if tk.has_key('meaning') and tk['meaning'].get('en'):
for ts in tk['meaning']['en']:
xnodes.append(py2neo.Node("Sense", sense=ts, lang="en"))
xrels.append(py2neo.Relationship(knode, "MEANS", xnodes[-1]))
# Readings (on-yomi, kun-yomi, nanori)
if tk.has_key('reading'):
if tk['reading'].has_key('ja_on'):
for tr in tk['reading']['ja_on']:
xnodes.append(py2neo.Node("Reading", reading=tr))
xrels.append(
py2neo.Relationship(knode,
"READS",
xnodes[-1],
yomi="on"))
if tk['reading'].has_key('ja_kun'):
for tr in tk['reading']['ja_kun']:
xnodes.append(py2neo.Node("Reading", reading=tr))
xrels.append(
py2neo.Relationship(knode,
"READS",
xnodes[-1],
yomi="kun"))
if tk['reading'].has_key('nanori'):
for tr in tk['reading']['nanori']:
xnodes.append(py2neo.Node("Reading", reading=tr))
xrels.append(
py2neo.Relationship(knode,
"READS",
xnodes[-1],
yomi="nanori"))
# Joyo
if tk.has_key('grade') and tk.has_key('jindex'):
xnodes.append(py2neo.Node("Joyo", joyo=int(tk['grade'])))
xrels.append(
py2neo.Relationship(xnodes[-1],
"SUBSET",
knode,
jindex=tk['jindex']))
# JLPT
if tk.has_key('jlpt') and isinstance(tk['jlpt'], int):
xnodes.append(py2neo.Node("Jlpt", jlpt=int(tk['jlpt'])))
xrels.append(py2neo.Relationship(xnodes[-1], "SUBSET", knode))
# SKIP
if tk.has_key('qcode') and tk['qcode'].has_key('skip'):
xnodes.append(py2neo.Node("Skip", skip=tk['qcode']['skip']))
xrels.append(py2neo.Relationship(knode, "WRITTEN", xnodes[-1]))
# Create Kanji node
try:
neo.create(knode)
except Exception as e:
logexc(e, u'Failed to create Kanji node')
# Create nodes
for tnode in xnodes:
try:
neo.create(tnode)
except Exception as e:
logexc(e, u'Failed to create aux node')
# Build relations
for trel in xrels:
# Check if Nodes are bound
sn = trel.start_node
en = trel.end_node
# if start node is not bound, then attempt a lookup
if not sn.bound:
nlab = list(sn.labels)[0]
nsn = neo.find_one(nlab, nlab.lower(), sn[nlab.lower()])
if nsn:
logthis(">>> Xref OK: %s '%s'" % (nlab, sn[nlab.lower()]),
loglevel=LL.DEBUG)
sn = nsn
# if end node is not bound, then attempt a lookup
if not en.bound:
elab = list(en.labels)[0]
nen = neo.find_one(elab, elab.lower(), en[elab.lower()])
if nen:
logthis(">>> Xref OK: %s '%s'" % (elab, en[elab.lower()]),
loglevel=LL.DEBUG)
en = nen
# Rebuild relationship
rrel = py2neo.Relationship(sn, trel.type, en, **trel.properties)
try:
neo.create_unique(rrel)
except Exception as e:
logexc(e, "Failed to build relationship")