def addMultiNodesFromJsonFile(neoGraph,filename):
file = open(filename)
# for all lines in the file
while 1:
line = file.readline()
if not line:
break
# replace '' with ""
respCmtJson = line.replace("'", "\"");
# ture string to json
js = simplejson.loads(respCmtJson)
# lable and key_values
labels = []
key_values = {}
#put ID into the dict
key_values['ID'] = js['ID']
keyValues = js['AVP_LIST']
# for all key_values in the AVP_LIST
for keyValue in keyValues:
# if ATTRIBUTE is category then put the value into label
if keyValue['ATTRIBUTE'] == 'entity_category':
labels = keyValue['VALUE'].split(',')
labels = tuple(labels)
#print len(labels)
else:
key_values[keyValue['ATTRIBUTE']] = keyValue['VALUE']
# labels is tuple and key_values is a dict
# use with which can create a node
node = Node()
neoGraph.create(node)
node.__init__(*labels,**key_values)
node.push()
createIndexofNode(neoGraph,node)
autoBuildRelationShip(neoGraph,node)
def mass_Import_WX_ID_from_opml(self, opemlFile_or_Content_or_URL):
'''
listparser.parse(obj[, agent, etag, modified])
Parameters:
obj (file or string) – a file-like object or a string containing a URL, an absolute or relative filename, or an XML document
agent (string) – User-Agent header to be sent when requesting a URL
etag (string) – ETag header to be sent when requesting a URL
modified (string or datetime) – Last-Modified header to be sent when requesting a URL
'''
opml = listparser.parse(opemlFile_or_Content_or_URL)
for feed in opml.feeds:
try:
wx_id=re.findall("weixin\?id=(\S+)$", feed.url)[0]
except IndexError:
print "---- WX_ID Paste Error!%s"%feed.url
if not self.is_WX_ID_Exists(wx_id):
WX_ID = Node("WX_ID")
info = {
"wx_id": wx_id,
"name": feed.title,
"group": feed.categories[0][0]
}
WX_ID.update(info)
self.neo4j.create(WX_ID)
print "++++ WX_ID Simple stored:\t%s" % wx_id
return True
def createNodeFromMapping(en,df):
node = Node()
for label in en['label']:
node.labels.add(label)
for i in range(len(en['graph'])):
node.properties[en['graph'][i]] = df[en['mysql'][i]][0]
return node, en['resolve']
def test_binding_node_if_labels_not_supported_casts_to_legacy_node():
with patch("py2neo.Graph.supports_node_labels") as mocked:
mocked.__get__ = Mock(return_value=False)
alice = Node(name="Alice Smith")
assert isinstance(alice, Node)
alice.bind("http://localhost:7474/db/data/node/1")
assert isinstance(alice, LegacyNode)
def test_can_pull_node(graph):
uri = graph.cypher.execute_one("CREATE (a {name:'Alice'}) RETURN a").uri
alice = Node()
alice.bind(uri)
assert alice.properties["name"] is None
batch = PullBatch(graph)
batch.append(alice)
batch.pull()
assert alice.properties["name"] == "Alice"
class GraphNode:
def __init__(self, node=None):
if node is None:
self.node = Node()
else:
self.node = node
def id(self):
"""
:return: node's unique Id
"""
return self.node._id
def labels(self):
"""
:return: labels set
"""
return self.node.labels
def degree(self):
"""
:return: number of relations
"""
return self.node.degree
def property(self, key):
"""
:param key: property name
:return: property value
"""
return self.node.properties[key]
def properties(self):
"""
:return: node's properties dict
"""
return self.node.properties
def relationships(self):
"""
:return: node's relationships iterator
"""
for rel in self.node.match():
i_rel = GraphRelation(rel)
yield i_rel
def outgoing_relationships(self):
"""
:return: node's outgoing relationships iterator
"""
for rel in self.node.match_outgoing():
i_rel = GraphRelation(rel)
yield i_rel
def test_rel_and_rev_hashes(graph):
assert hash(Rel("KNOWS")) == hash(Rel("KNOWS"))
assert hash(Rel("KNOWS")) == -hash(Rev("KNOWS"))
assert hash(Rel("KNOWS", well=True, since=1999)) == hash(Rel("KNOWS", since=1999, well=True))
rel_1 = Node("KNOWS", since=1999)
graph.create(rel_1)
rel_2 = Node("KNOWS", since=1999)
rel_2.bind(rel_1.uri)
assert rel_1 is not rel_2
assert hash(rel_1) == hash(rel_2)
def create_WX_ID(self, wx_id_info):
'''
CREATE (TheMatrix:Movie {title:'The Matrix', released:1999, tagline:'Welcome to the Real World'})
CREATE (Hugo:Person {name:'Hugo Weaving', born:1960})
'''
if not self.is_WX_ID_Exists(wx_id_info['wx_id']):
wxid1 = Node("WX_ID")
wxid1.update(wx_id_info)
self.neo4j.create(wxid1)
print "++++ WX_ID stored:\t%s" % wx_id_info['wx_id']
return True
def test_node_hashes(graph):
assert hash(Node()) == hash(Node())
assert hash(Node(name="Alice")) == hash(Node(name="Alice"))
assert hash(Node(name="Alice", age=33)) == hash(Node(age=33, name="Alice"))
assert hash(Node("Person", name="Alice", age=33)) == hash(Node("Person", age=33, name="Alice"))
node_1 = Node("Person", name="Alice")
graph.create(node_1)
node_2 = Node("Person", name="Alice")
node_2.bind(node_1.uri)
assert node_1 is not node_2
assert hash(node_1) == hash(node_2)
def test_node_cast():
alice = Node("Person", "Employee", name="Alice", age=33)
assert Node.cast() == Node()
assert Node.cast(None) is None
assert Node.cast(alice) is alice
assert Node.cast("Person") == Node("Person")
assert Node.cast(name="Alice") == Node(name="Alice")
assert Node.cast("Person", "Employee", name="Alice", age=33) == alice
assert Node.cast({"name": "Alice"}) == Node(name="Alice")
assert Node.cast(("Person", "Employee", {"name": "Alice", "age": 33})) == alice
assert Node.cast(42) == NodePointer(42)
assert Node.cast(NodePointer(42)) == NodePointer(42)
def make_node(nodetype,uid,name,properties=None,property_key="id"):
node = None
if graph.find_one(nodetype,property_key='id', property_value=uid) == None:
print("Creating %s:%s, %s" %(nodetype,name,uid))
timestamp = graph.cypher.execute("RETURN timestamp()").one
node = Node(nodetype, name=name,id=uid,creation_time=timestamp,last_updated=timestamp)
graph.create(node)
if properties != None:
for property_name in properties.keys():
node.properties[property_name] = properties[property_name]
node.push()
return node
def get_product_details(self,product_url,gender):
print self.count
self.count = self.count + 1
try:
product_html = requests.get(product_url).content
except Exception as e:
product_html = requests.get(product_url).content
product_soup = BeautifulSoup(product_html,"lxml")
product_image = product_soup.find("img","articleMedia_imagePlaceholder")
article_span = product_soup.find("span","sku")
product_name = product_soup.find("h1","productName noBg")
brand = product_name.contents[1]
product_info = dict()
product_info["article_number"] = article_span.string
product_info["brand"] = brand.string
product_info["gender"] = gender
ul = article_span.parent.parent
self.logger.write("%s being explored" %(article_span.string)+"\n")
for li in ul.contents:
if li.string is not None and str(type(li))=="<class 'bs4.element.Tag'>":
var = li.get('class')[0]
if(len(li.string.split(":")) > 1 ):
product_info[li['class'][0]] = li.string.split(":")[1].encode("ascii","ignore")
n = Node(gender,"Zalando","Product")
for key in product_info:
n.properties[key] = product_info[key]
while (product_image is not None):
url = product_image['src']
file_name = url.split('/')[-1]
folder_name = file_name.split('@')[0]
if os.path.exists(folder_name):
break
os.makedirs(folder_name)
os.chdir(folder_name)
url = url.replace("detail","large")
try:
r = requests.get(url)
except Exception as e:
r = requests.get(url)
open(file_name,"w").write(r.content)
os.chdir("..")
n["file_system_url"] = "Zalando/"+folder_name
n["image_url"] = url
product_image = product_image.nextSibling.nextSibling
try:
self.zalando_graph.create(n)
except Exception as e:
self.logger.write("Node %s already exists" %(article_span.string))
def test_can_pull_node_with_label(graph):
if not graph.supports_node_labels:
return
uri = graph.cypher.execute_one("CREATE (a:Person {name:'Alice'}) RETURN a").uri
alice = Node()
alice.bind(uri)
assert "Person" not in alice.labels
assert alice.properties["name"] is None
batch = PullBatch(graph)
batch.append(alice)
batch.pull()
assert "Person" in alice.labels
assert alice.properties["name"] == "Alice"
def test_node_exists_will_raise_non_404_errors():
with patch.object(_Resource, "get") as mocked:
error = GraphError("bad stuff happened")
error.response = DodgyClientError()
mocked.side_effect = error
alice = Node(name="Alice Smith")
alice.bind("http://localhost:7474/db/data/node/1")
try:
_ = alice.exists
except GraphError:
assert True
else:
assert False
def create_node(self, node_label, node_dict):
cur_nid = self.get_max_nid() + 1
node_dict[ArianeDefinitions.GRAPH_NODE_ID] = cur_nid
node = Node.cast(node_dict)
node.labels.add(node_label)
self.graph.create(node)
return node, cur_nid
def create_company_node(name, tw=None, em=None, fb=None):
""" Creates a company type node
Parameters: Name, Twitter Handle, Email Address, Facebook URL
Returns: Company Node
"""
properties = {"name" : name}
properties["facebook"] = [fb] if fb else []
properties["twitter"] = [tw] if tw else []
properties["email"] = [em] if em else []
company = Node.cast("company", properties)
# searches for company nodes with the same name
results = list(graph.find("company", "name", name))
if results:
# if there are already nodes of the same name in the graph,
# add the information from the incoming node to the duplicate
dupe = results[0]
if fb and not fb in dupe["facebook"]: dupe["facebook"].append(fb)
if tw and not tw in dupe["twitter"]: dupe["twitter"].append(tw)
if em and not em in dupe["email"]: dupe["email"].append(em)
dupe.push()
else:
# if there are no nodes of the same name in the graph,
# add the company node to the graph as it is
graph.create(company)
def peuple(self, commune):
voiesdelacommune = self.voies[commune]
nbvoiesdelacommune = len(self.voies[commune])
placecommune = list(self.communes).index(commune)+1
nbcommunes = len(self.communes)
cpt = 0
for v in voiesdelacommune:
#if not re.compile(".*D59.*").match(v.libelle): continue
numero = Numero(7, v.libelle, v.sti, v.rivoli[:-1], v.rivoli, None, None)
cpt += 1
## Pour contourner un bug d'espace dans le fichier d'open data..
if numero.lib_off[0] == " ": numero.lib_off = numero.lib_off[1:]
try:
## On recupere le noeud de rue qui va recevoir ce numero. Ne retourne normalement qu'une rue donc on selectionne le premier et unique resultat avec ,
#voie, = self.graph.cypher.execute( "match (v:Voie { libelle:'%s' })-[:ORGANISE]-(c:Commune {libcom:'%s'}) return v" % (numero.lib_off, commune) )
voie, = self.graph.cypher.execute( "match (v:Voie { libelle:'%s' })-[:ORGANISE]-(c:Commune {libcom:'%s'}) return v" % (numero.lib_off, commune) )
except ValueError as e:
print("WARNING;%s; La voie <%s> de la commune <%s> n'existe pas." % (cpt,numero.lib_off,commune) )
print("%s" % (e) )
continue
for n in self.nums_dans_rue:
numero.no = n
newnum = Node.cast("Numero", numero.to_dict() )
#print( " %s ; %s ; %s\n%s" % (n,type(newnum),newnum,voie) )
## Le noeud Numero est crée en meme temps que la relation
arete = self.graph.create( Relationship(voie['v'],"COMPORTE",newnum) )
#if n % 50 == 0 : print( "%s ; %s ; %s" % (n,type(newnum),newnum) )
#if n % 50 == 0 : print( " %s" % (arete) )
if cpt %10 == 0 : print( "Voie %s/%s de la commune %s / %s" % (cpt,nbvoiesdelacommune, placecommune, nbcommunes) )
def test_full_node_hydrate():
dehydrated = {
"extensions": {
},
"paged_traverse": "http://localhost:7474/db/data/node/0/paged/traverse/{returnType}{?pageSize,leaseTime}",
"labels": "http://localhost:7474/db/data/node/0/labels",
"outgoing_relationships": "http://localhost:7474/db/data/node/0/relationships/out",
"traverse": "http://localhost:7474/db/data/node/0/traverse/{returnType}",
"all_typed_relationships": "http://localhost:7474/db/data/node/0/relationships/all/{-list|&|types}",
"property": "http://localhost:7474/db/data/node/0/properties/{key}",
"all_relationships": "http://localhost:7474/db/data/node/0/relationships/all",
"self": "http://localhost:7474/db/data/node/0",
"outgoing_typed_relationships": "http://localhost:7474/db/data/node/0/relationships/out/{-list|&|types}",
"properties": "http://localhost:7474/db/data/node/0/properties",
"incoming_relationships": "http://localhost:7474/db/data/node/0/relationships/in",
"incoming_typed_relationships": "http://localhost:7474/db/data/node/0/relationships/in/{-list|&|types}",
"create_relationship": "http://localhost:7474/db/data/node/0/relationships",
"data": {
"name": "Alice",
"age": 33,
},
}
hydrated = Node.hydrate(dehydrated)
assert isinstance(hydrated, Node)
assert hydrated.properties == dehydrated["data"]
assert hydrated.bound
assert hydrated.resource.uri == dehydrated["self"]
def infect(self):
# Deploys new version of Khan Academy to this user
self.khanAcademyVersion = 'B'
######################DATABASE#######################
# save all relationships, ingoing and outgoing of this node
incomingRel = self.databaseNode.match_incoming(rel_type="IS_COACHING")
outgoingRel = self.databaseNode.match_outgoing(rel_type="IS_COACHING")
# create a node that is of typer UserN
self.databaseNode = Node("UserB", name = self.name, khanAcademyVersion = self.khanAcademyVersion, LastLogin = self.lastLogin)
graph.create(self.databaseNode)
# Build the same relationships as the UserA node
for r in incomingRel:
startNode = r.start_node
graph.create(Relationship(startNode, "IS_COACHING", self.databaseNode))
for r in outgoingRel:
endNode = r.end_node
graph.create(Relationship(self.databaseNode, "IS_COACHING", endNode))
# Delete the old node
cypher.execute("MATCH (u :UserA {khanAcademyVersion: {khanAcademyVersion}, name: {name}})-[r]-() DELETE r", khanAcademyVersion = 'A', name = self.name)
cypher.execute("MATCH n WHERE (n.khanAcademyVersion = {khanAcademyVersion} AND n.name = {name}) DELETE n", khanAcademyVersion = 'A', name = self.name)
def save_publications_list(publications):
"""
Method to save all the publications to neo4j database
"""
graph = Graph()
for publication in publications:
publication_node = Node("Publication")
for key in publication:
if key not in ['_id', 'authorsSearched', 'identifiers', 'rank', 'work-citation-type']:
publication_node.properties[key] = publication[key]
graph.create(publication_node)
print 'Inserted Publication: ' + publication['doi']
def test_minimal_node_hydrate():
dehydrated = {
"self": "http://localhost:7474/db/data/node/0",
}
hydrated = Node.hydrate(dehydrated)
assert isinstance(hydrated, Node)
assert hydrated.bound
assert hydrated.resource.uri == dehydrated["self"]
def create(self,name,properties=None,property_key="id"):
'''create will create a new node of nodetype with unique id uid, and properties
:param uid: the unique identifier
:param name: the name of the node
:param properties: a dictionary of properties with {field:value} for the node
'''
node = None
uid = generate_uid(self.name) # creation also checks for existence of uid
if graph.find_one(self.name,property_key='id',property_value=uid) == None:
timestamp = graph.cypher.execute("RETURN timestamp()").one
node = NeoNode(self.name, name=name,id=uid,creation_time=timestamp,last_updated=timestamp)
graph.create(node)
if properties != None:
for property_name in properties.keys():
node.properties[property_name] = properties[property_name]
node.push()
return node
def create_node(self,graph_db):
# Ajouter propriétés du type "modified" ?
fiche_properties = {'doc_position': self.tmp_id, 'titre': self.titre,
'auteur': self.auteur, 'contenu': self.contenu, 'date_creation': self.date_creation}
fiche_node = Node.cast(fiche_properties)
fiche_node.labels.add(self.node_type)
self._node = fiche_node
graph_db.create(self._node)
def get_character(message): if message['AuctionType'] == 'request': char_id = message['BuyerCharacterId'] char_name = message['BuyerName'] elif message['AuctionType'] == 'offer': char_id = message['SellerCharacterId'] char_name = message['SellerName'] char_node = Node( 'Character', Id = char_id, Name = char_name ) char_node.__primarylabel__ = 'Character' char_node.__primarykey__ = 'Id' return char_node
def add_nodes(xml_blob):
''' Add each Scappe "note" as a neo4j node '''
node = Node('item', uid=int(xml_blob.get('ID')))
for item in xml_blob.getchildren():
if item.tag == 'String':
node.properties['text'] = item.text
elif item.tag == 'PointsToNoteIDs':
# format is either '1, 2, 3' or '1-3'
link_range = item.text.split(', ')
links = []
for value in link_range:
if '-' in value:
value = value.split('-')
links += range(int(value[0]), int(value[1])+1)
else:
links.append(int(value))
node.properties['pointsTo'] = links
GRAPH.create(node)
def construct_node(self, push_namespace, relationship, label, name, properties):
namespace = self.get_namespace(name)
self.check_duplicates(label, name=namespace)
node = Node(label)
node.properties["namespace"] = namespace
node.properties["name"] = name
for i in properties.keys():
node.properties[i] = properties[i]
self.graph.create(node)
if len(self.parent_node) != 0:
relation_enity = Relationship(self.get_parent_node(), relationship, node)
self.graph.create(relation_enity)
if push_namespace == True:
self.namespace.append(name)
self.parent_node.append(node)
def get_item(message):
group = message['ItemGroupTypeId']
enchantment = message['EnchantmentLevel']
quality = message['QualityLevel']
item_id = f"{group}@{enchantment}&{quality}"
item_node = Node(
'Item',
Id = item_id,
Group = message['ItemGroupTypeId'],
Tier = message['Tier'],
Enchantment = enchantment,
Quality = quality
)
item_node.__primarylabel__ = "Item"
item_node.__primarykey__ = 'Id'
return item_node
def load_entities_csv(g,filename, label,prop_list):
#Note prop_list must contain respective column names from the dataset
df = pd.read_csv(filename)
print "[load_entities_csv] - df given:-"
print df
for i,r in df.iterrows():
print '[load_entities_csv] - scanning row - {}'.format(i)
nd = Node()
nd.labels.add(label)
for c in prop_list:
nd.properties[c] = r[c]
print '[load_entities_csv] - creating node...'
g.create(nd)
print '[load_entities_csv] - node successfully created'
return
def create_cq(user_node, cq_dict):
cq_dict['id'] = str(uuid.uuid4())
cq_dict['created_date'] = datetime.date.today()
cq_node = Node.cast(GraphLabel.CQ,
cq_dict)
cq_node, = Graph(settings.DATABASE_URL).create(cq_node)
cq_relationship = Relationship(user_node,
GraphRelationship.SENT,
cq_node)
Graph(settings.DATABASE_URL).create_unique(cq_relationship)
def upsert_user(user):
"""
:param user: dictionary
:return:
"""
drenaj_api_logger.info('Upserting: ' + user['id_str'])
# for key in user.keys():
# print key + ': ' + str(user[key]) + " - " + str(type(user[key]))
# removing these beacuse neo4j doesn't allow nested nodes.
if 'entities' in user:
user['entities'] = bson.json_util.dumps(user['entities'])
if 'status' in user:
del user['status']
# Find the related user. This looks complicated because of {'id_str': '', 'screen_name': 'blabla'} entries
# coming from create_campaign
user_node = None
if 'id_str' in user and 'screen_name' in user and user['id_str'] != '' and user[
'screen_name'] != '':
user_node = graph.cypher.execute("MATCH (u:User) WHERE u.id_str = {id_str} RETURN u",
{'id_str': user['id_str']}).one
if not user_node:
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.screen_name = {screen_name} RETURN u",
{'screen_name': user['screen_name']}).one
# TODO: This is very nasty! Go get learn the proper way!
elif (type(user['id_str']) == type('STRING') or type(user['id_str']) == type(u'STRING')) and \
user['id_str'] != '':
user_node = graph.cypher.execute("MATCH (u:User) WHERE u.id_str = {id_str} RETURN u",
{'id_str': user['id_str']}).one
elif (type(user['screen_name']) == type('STRING') or type(user['screen_name']) == type(
u'STRING')) and user['screen_name'] != '':
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.screen_name = {screen_name} RETURN u",
{'screen_name': user['screen_name']}).one
print user_node
if user_node:
# First clearing the user_node properties.
for key in user_node.properties.keys():
user_node.properties[key] = None
# then assigning the new properties
for key in user.keys():
user_node.properties[key] = user[key]
user_node.push()
else:
user_node = Node.cast(user)
user_node.labels.add("User")
graph.create(user_node)
return user_node
def init(person_first_name, person_last_name, attended_university_name, degree,
subject, graduation_date):
Person_node = Node(label='Person', )
def _to_neo(p_info, g_info, level, graph):
print(p_info['OriginalTitle'])
tx = graph.begin()
paper = Node("Paper", CellID=p_info['CellID'])
citation_paper = [
Node("Paper", CellID=pid, level=level + 1)
for pid in p_info['CitationIDs']
]
# reference_paper = [Node("Paper", CellID=pid, level=level+1) for pid in p_info['ReferenceIDs']]
citations = [Relationship(p, 'CITED', paper) for p in citation_paper]
# references = [Relationship(paper, 'CITED', p) for p in reference_paper]
# fields_of_study = [Node("FieldOfStudy", CellID=fid) for fid in p_info['FieldOfStudyIDs']]
# paper_fos = [Relationship(paper, 'IS_ABOUT', fos) for fos in fields_of_study]
# keywords = [Node("Keyword", CellID=hash(kw), name=kw) for kw in p_info['Keywords']]
# paper_kw = [Relationship(paper, 'IS_ABOUT', kw) for kw in keywords]
paper_author = [
Node("PaperAuthor", CellID=hash(aid + p_info['CellID']))
for aid in p_info['AuthorIDs']
]
authors = [Node("Author", CellID=aid) for aid in p_info['AuthorIDs']]
affilitation = [
Node("Affiliation", CellID=aid) for aid in p_info['AffiliationIDs']
]
authorships1 = [Relationship(paper, 'HAS', pa) for pa in paper_author]
authorships2 = [
Relationship(pa, 'IS', a) for pa, a in zip(paper_author, authors)
]
affiliationship = [
Relationship(pa, 'AFFILIATE', aff)
for pa, aff in zip(paper_author, affilitation)
]
# nodes = citation_paper + reference_paper + [paper] + fields_of_study + keywords + paper_author + authors
# relations = authorships1 + authorships2 + affiliationship + citations + references + paper_fos + paper_kw
nodes = [paper] + paper_author + authors + citation_paper + affilitation
relations = authorships1 + authorships2 + affiliationship + citations
sg = Subgraph(nodes, relations)
tx.merge(sg, primary_label=None, primary_key='CellID')
tx.commit()
direct_info = keyfilter(direct.__contains__, p_info)
if g_info:
google_info = {f'google_{k}': v for k, v in g_info.items()}
else:
google_info = {}
paper.update(**direct_info, **google_info, level=level)
graph.push(paper)
for aff, name in zip(affilitation, p_info['NormalizedAffiliations']):
aff.update(NormalizedName=name)
graph.push(aff)
for pa, seq_number in zip(paper_author, p_info['AuthorSequenceNumbers']):
pa.update(AuthorSequenceNumbers=seq_number)
graph.push(pa)
import py2neo
#graph直接写账号密码会不安全
g = Graph('http://localhost:7474', user='******', password='******')
#创建config以及db.cfg用来存储信息
with open('/Users/ren/PycharmProjects/基于知识图谱的电影问答系统/data/genre.csv',
'r',
encoding='utf-8') as f:
#数据集除了第一行代表属性外，第一列为实体1，第二列为实体2，第三列是两者英文关系，第四列为两者中文关系
reader = csv.reader(f)
for item in reader:
#第一行的标签不是咱们需要的内容，line_num表示文件的第几行
if reader.line_num == 1:
continue
print("当前行数：", reader.line_num, "当前内容", item)
test_node_1 = Node("Genre", id=item[0], name=item[1])
g.merge(test_node_1, "Genre", "id")
with open('/Users/ren/PycharmProjects/基于知识图谱的电影问答系统/data/movie.csv',
'r',
encoding='utf-8') as f:
reader = csv.reader(f)
for item in reader:
#第一行的标签不是咱们需要的内容，line_num表示文件的第几行
if reader.line_num == 1:
continue
print("当前行数：", reader.line_num, "当前内容", item)
test_node_1 = Node("Movie",
id=item[0],
title=item[1],
introduction=item[2],
rating=item[3],
from py2neo import Graph, Node
graph = Graph(password="******")
alice = Node("Person", name="Alice")
graph.create(alice)
german, speaks = graph.create({"name": "German"}, (alice, "SPEAKS", 0))
# later this could be improved by bulk upload mechanisms e.g. from replica db dump
from enes_settings import *
from py2neo import authenticate, Node, Relationship, Graph
authenticate("localhost:7474", 'neo4j', 'prolog16')
graph = Graph("http://localhost:7474/db/data/")
from pyesgf.search import SearchConnection
#SEARCH_SERVICE = 'http://pcmdi9.llnl.gov/esg-search'
SEARCH_SERVICE = 'http://esgf-data.dkrz.de/esg-search'
conn = SearchConnection(SEARCH_SERVICE,distrib=True)
project = u'CMIP5'
ctx = conn.new_context(project=project,replica=True,latest=True)
project_node = Node("Collection",name='cmip5',level=10, identifier = 'project')
try:
products
except NameError:
products = conn.new_context(project=project,replica=True,latest=True).facet_counts['product'].keys()
for product in products:
product_node = Node("Collection",name=product, level=9)
rel = Relationship(product_node,"belongs_to",project_node)
graph.create(rel)
#for institute in facet_counts['institute']:
print products
try:
institutes
except NameError:
def create_kg(self):
graph = Graph("http://localhost:7474",
username="******",
password='******')
matcher = NodeMatcher(graph)
graph.delete_all()
for i in self.id_entity_relation_attribute:
id_node = matcher.match('ID').where(name=str(i[0])).first()
if id_node == None:
id_node = Node('ID', name=str(i[0]))
self.num_entity['ID'] += 1
graph.create(id_node)
entity = i[1]
flag = 0
for dict in self.dict_list:
entity_change = entity.replace('-C', '').replace(
'-P', '') # 先将这些标签去掉，看看有没有，如果没有在下面在判断
if entity_change == dict[0]:
flag = 1
entity_node = matcher.match(
dict[1]).where(name=str(entity)).first()
if entity_node == None:
entity_node = Node(dict[1], name=dict[0])
if dict[1] not in self.num_entity.keys():
self.num_entity[dict[1]] = 1
else:
self.num_entity[dict[1]] += 1
graph.create(entity_node)
relation = Relationship(id_node,
i[2],
entity_node,
property=i[3])
graph.create(relation)
if i[2] not in self.num_relation.keys():
self.num_relation[i[2]] = 1
else:
self.num_relation[i[2]] += 1
break
if flag == 0:
if i[2] in ['儿童用药', '老年用药',
'妇女用药']: # 在之前人群中没有出现过的和儿童妇女老人相关的描述，
entity_node = Node('病人属性', name=i[1])
self.num_entity['病人属性'] += 1
graph.create(entity_node)
relation = Relationship(id_node,
i[2],
entity_node,
property=i[3])
graph.create(relation)
if i[2] not in self.num_relation.keys():
self.num_relation[i[2]] = 1
else:
self.num_relation[i[2]] += 1
elif '-P' in entity: # 标注过程中标记的病人标签
entity_node = matcher.match('病人属性').where(
name=str(entity.replace('-P', ''))).first()
if entity_node == None:
entity_node = Node('病人属性',
name=entity.replace('-P', ''))
if '病人属性' not in self.num_entity.keys():
self.num_entity['病人属性'] = 1
else:
self.num_entity['病人属性'] += 1
relation = Relationship(id_node,
i[2],
entity_node,
property=i[3])
graph.create(relation)
if i[2] not in self.num_relation.keys():
self.num_relation[i[2]] = 1
else:
self.num_relation[i[2]] += 1
elif '-C' in entity or i[2] == '包含': # 标记过程中标记的成份标签
entity_node = matcher.match('西药成份').where(
name=str(entity.replace('-C', ''))).first()
if entity_node == None:
entity_node = Node('西药成份',
name=entity.replace('-C', ''))
if '西药成份' not in self.num_entity.keys():
self.num_entity['西药成份'] = 1
else:
self.num_entity['西药成份'] += 1
relation = Relationship(id_node,
i[2],
entity_node,
property=i[3])
graph.create(relation)
if i[2] not in self.num_relation.keys():
self.num_relation[i[2]] = 1
else:
self.num_relation[i[2]] += 1
elif i[2] == '治疗':
entity_node = matcher.match('症状').where(
name=entity).first()
if entity_node == None:
entity_node = Node('症状', name=entity)
if '症状' not in self.num_entity.keys():
self.num_entity['症状'] = 1
else:
self.num_entity['症状'] += 1
relation = Relationship(id_node,
'治疗',
entity_node,
property=i[3])
graph.create(relation)
if '治疗' not in self.num_relation.keys():
self.num_relation['治疗'] = 1
else:
self.num_relation['治疗'] += 1
elif i[3] == '检查':
# 之前没有出现过的一次特殊的实体，例如 关系属性为 检查 时，实体对应的都是些体征和检查手段，在之前的字典中是没有的
entity_node = matcher.match('检查手段').where(
name=str(entity)).first()
if entity_node == None:
entity_node = Node('检查手段', name=entity)
if '检查手段' not in self.num_entity.keys():
self.num_entity['检查手段'] = 1
else:
self.num_entity['检查手段'] += 1
if '检查手段' not in self.num_relation.keys():
self.num_relation[i[2]] = 1
else:
self.num_relation[i[2]] += 1
relation = Relationship(id_node,
i[2],
entity_node,
property=i[3])
graph.create(relation)
else:
# print('在字典中找不到实体分类',entity)
if entity not in self.wrong_entity:
self.wrong_entity.append(entity)
zong = 0
for i in self.num_entity.values():
zong += i
print('在好心情知识图谱中实体共有{}个，具体数量如下\n实体|数量\n:-:|:-:'.format(zong))
for k, v in self.num_entity.items():
print(k, '|', v)
print('\n')
zong = 0
for i in self.num_relation.values():
zong += i
print('在好心情知识图谱中关系共有{}个，具体数量如下\n关系类别|数量\n:-:|:-:'.format(zong))
for k, v in self.num_relation.items():
print(k, '|', v)
print('\n')
def test_get_node_by_id(graph):
a1 = Node(foo="bar")
graph.create(a1)
a2 = graph.nodes.get(a1.identity)
assert a1 == a2
def test_create_single_empty_node(graph):
a = Node()
graph.create(a)
assert a.graph == graph
assert a.identity is not None
def test_can_get_node_by_id(graph):
node = Node()
graph.create(node)
got = graph.nodes.get(node.identity)
assert got == node
def add_subspecies_nodes(self, node, fields):
for s in range(0, len(fields)):
#亚种节点添加属性和从属的器官节点
if fields[s][0][0] == "subspecies":
#创建亚种节点
subspecies_node = Node("subspecies", 中文名=fields[s][0][1])
#添加异名和拉丁名属性
synonyms = []
for i in range(1, len(fields[s])):
if fields[s][i][0] == "yiming":
synonyms.append(fields[s][i][1])
if fields[s][i][0] == "latin":
latinname = node["拉丁学名"] + " " + fields[s][i][1]
if synonyms != []:
subspecies_node["异名"] = synonyms
subspecies_node["拉丁学名"] = latinname
if fields[s][0][0] == "lifeform": #有生活型tag的行
#添加生活型属性
subspecies_node["生活型"] = fields[s][0][1]
#如果有性状tag，添加性状属性
if len(fields[s]) > 1:
for i in range(1, len(fields[s])):
if fields[s][i][0] == "sx":
subspecies_node[fields[s][i][1]] = fields[s][i +
1][1]
if fields[s][0][0] == "sj":
subspecies_node["生境"] = fields[s][0][1]
if fields[s][0][0] == "fenbu":
subspecies_node["国内分布"] = fields[s][0][1]
if fields[s][0][0] == "guowaifenbu":
subspecies_node["国外分布"] = fields[s][0][1]
if re.match(r"[hg]u[ao]qi", fields[s][0][0]) != None: #有花期/果期tag的行
for i in range(0, len(fields[s])):
if fields[s][i][0] == "guoqi":
subspecies_node["果期"] = fields[s][i][1]
if fields[s][i][0] == "huaqi":
subspecies_node["花期"] = fields[s][i][1]
if fields[s][0][0] == "qiguan": #有器官tag的行，创建器官节点
n = len(fields[s])
organ_node = Node("organ",
中文名=fields[s][0][1],
原文=fields[s][n - 1][1])
r = Relationship(organ_node, "part_of", subspecies_node)
for i in range(1, len(fields[s])):
if fields[s][i][0] == "sx":
organ_node[fields[s][i][1]] = fields[s][i + 1][1]
self.graph_plants.create(organ_node | r | subspecies_node)
for i in range(1, len(fields[s])): #一句话描述两种器官的情况
if fields[s][i][0] == "器官":
organ_node2 = Node("organ",
中文名=fields[s][i][1],
原文=fields[s][n - 1][1])
r2 = Relationship(organ_node2, "part_of",
subspecies_node)
self.graph_plants.create(organ_node2 | r2
| subspecies_node)
self.graph_plants.push(subspecies_node) #更新节点
r = Relationship(subspecies_node, "belong_to",
node) #建立亚种-[从属于]->物种的关系
self.graph_plants.create(r)
def test_create_node_with_null_properties(graph):
a = Node(foo="bar", no_foo=None)
graph.create(a)
assert a["foo"] == "bar"
assert a["no_foo"] is None
def insert(self):
YD = YelpData()
row_sum = 10876672
# row_sum = 10
read_unit = 100
rest_sum = row_sum - row_sum / read_unit * read_unit
record_point = 2971
for i in range(record_point, row_sum / read_unit + 1):
# reviews = YD.read_reviews_limit(1, 10)
self.tx = self.neo4j_graph.begin()
if i == row_sum / read_unit:
print 10876672
reviews = YD.read_reviews_limit(i * read_unit, rest_sum)
else:
print(i + 1) * read_unit
reviews = YD.read_reviews_limit(i * read_unit, read_unit)
# reviews = self.read_reviews_limit(i*10, 10)
for reviews_data in reviews:
review_id = reviews_data[0]
bussiness_id = reviews_data[1]
reviewer_name = reviews_data[2]
reviewer_id = reviews_data[3]
urating = reviews_data[4]
udate = reviews_data[5]
ucontent = reviews_data[6]
prating = reviews_data[7]
pdate = reviews_data[8]
pcontent = reviews_data[9]
pnum = reviews_data[10]
fake = reviews_data[11]
# # 编码处理
# ucontent_encoding = chardet.detect(ucontent)['encoding']
# # print ucontent_encoding
# ucontent_decoded = ucontent.decode(ucontent_encoding).encode('utf8')
# # print chardet.detect(ucontent_decoded)
# pcontent_encoding = chardet.detect(pcontent)['encoding']
# # print pcontent_encoding
# pcontent_decoded = pcontent.decode(pcontent_encoding).encode('utf8')
# # print chardet.detect(pcontent_decoded)
# if reviewer_name is not "": # 名字可能为空
# reviewer_name_encoding = chardet.detect(reviewer_name)['encoding']
# reviewer_name_decoded = reviewer_name.decode(reviewer_name_encoding).encode('utf8')
# else:
# reviewer_name_decoded = ""
# construct relationship
reviewer_node = Node('reviewer',
id=reviewer_id,
reviewer_name=reviewer_name,
udate=udate)
self.tx.merge(reviewer_node, 'reviewer', "id")
review_node = Node('review',
id=review_id,
ucontent=ucontent,
urating=urating,
fake=fake)
self.tx.merge(review_node,
primary_label='review',
primary_key="id")
product_node = Node('product',
id=bussiness_id,
prating=prating,
pdate=pdate,
pcontent=pcontent,
pnum=pnum)
self.tx.merge(product_node,
primary_label='product',
primary_key="id")
reviewer_review = Relationship(reviewer_node, 'publish',
review_node)
review_product = Relationship(review_node, 'comment',
product_node)
# if self.tx.exists(reviewer_review):
# print "not exists rr"
self.tx.merge(reviewer_review)
# if self.tx.exists(review_product):
# print "not exists rp"
self.tx.merge(review_product)
# self.tx.push(reviewer_review)
# self.tx.push(review_product)
print time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(time.time()))
self.tx.commit()
from py2neo import Graph, Node, Relationship
g = Graph(host="localhost") #password="******")
tx = g.begin()
john = Node("User", name="John", gender="M", age="28")
tx.create(john)
mary = Node("User", name="Mary", gender="F", age="26")
tx.create(mary)
jm = Relationship(john, "FRIENDS_WITH", mary)
tx.create(jm)
tx.commit()
graph2 = Graph("http://121.36.99.228:7474", auth=("neo4j", "SSPKUsspku12345"))
data = graph2.run(
"MATCH (m:configuration_file)-[r]->(n:configuration_file) where m.name='configuration_file' RETURN n"
).data()
hadoop_node = graph.nodes.match("技术", name="hadoop").first()
yarn_node = graph.nodes.match("技术", name="yarn").first()
for ele in data:
new_data = build_nodes(ele)
# print(new_data['name'])
com_node = Node("配置文件",
name=new_data['name'],
detail=new_data['detail'],
platform='hadoop',
path=new_data['path'])
ab2 = Relationship(hadoop_node, "技术有关配置文件", com_node)
graph.create(ab2)
data2 = graph2.run(
"MATCH (m:component)-[r]->(n:configuration_file) where m.name='yarn' RETURN n"
).data()
for ele in data2:
new_data2 = build_nodes(ele)
com_node2 = Node("配置文件",
name=new_data2['name'],
detail=new_data2['detail'],
platform='hadoop',
def fileGraph():
fileAnalysis = MainNode()
compFiles = CompFiles()
exFilesdf = pd.read_csv(fileExPath)
for index, row in exFilesdf.iterrows():
compFiles.add_files(
ExFiles(row['Title'], row['Time'], row['FullPath'],
row['LengthInKB'], row['HexSignature'], row['FileExtn'],
row['Result']))
fileAnalysis.add_component(compFiles)
return fileAnalysis
if __name__ == "__main__":
graph = DFGraph()
user = Node("User", name=userAcc)
graph.add_node(user)
registry = registryGraph()
swReg = Node("Registry", name="Software Installed")
mruReg = Node("Registry", name="Most Recently Used")
graph.add_node(Relationship(user, "Digital Traces By", swReg))
graph.add_node(Relationship(user, "Digital Traces By", mruReg))
for softwares in registry.components[0].softwares:
graph.add_node(
Relationship(
swReg, "Extracted Info",
Node("Softwares",
name=softwares.name,
version=softwares.version,
publisher=softwares.publisher,
def create(cls):
name = Node(cls.attribute, name="名称", name2="名称")
cls.graph.create(name)
variant_name = Node(cls.attribute, name="别名", name2="别名")
cls.graph.create(variant_name)
weekdy_opentime = Node(cls.attribute, name="开放时间", name2="开放时间")
cls.graph.create(weekdy_opentime)
open_date = Node(cls.attribute, name="开放日", name2="开放日")
cls.graph.create(open_date)
#workday_opentime = Node(cls.attribute, name="工作日开放时间")
#cls.graph.create(workday_opentime)
phone = Node(cls.attribute, name="电话", name2="电话")
cls.graph.create(phone)
describe = Node(cls.attribute, name="描述", name2="描述")
cls.graph.create(describe)
#weekdy_btime = Node(cls.attribute, name="周末借阅时间")
#cls.graph.create(weekdy_btime)
workday_btime = Node(cls.attribute, name="借阅时间", name2="借阅时间")
cls.graph.create(workday_btime)
borrow = Node(cls.attribute, name="借阅", name2="借阅")
cls.graph.create(borrow)
position = Node(cls.attribute, name="位置", name2="位置")
cls.graph.create(position)
site = Node(cls.attribute, name="坐标", name2="坐标")
cls.graph.create(site)
time = Node(cls.attribute, name="时间", name2="时间")
cls.graph.create(time)
num = Node(cls.attribute, name="数量", name2="数量")
cls.graph.create(num)
age = Node(cls.attribute, name="年龄", name2="年龄")
cls.graph.create(age)
function = Node(cls.attribute, name="功能", name2="功能")
cls.graph.create(function)
#############################################################
library = Node(cls.library, name="国家图书馆")
cls.graph.create(library)
room = Node(cls.room, name="馆室")
cls.graph.create(room)
area = Node(cls.building, name="馆区")
cls.graph.create(area)
floor = Node(cls.floor, name="楼层")
cls.graph.create(floor)
card = Node(cls.card, name="读者卡")
cls.graph.create(card)
resource = Node(cls.resource, name="资源")
cls.graph.create(resource)
type = Node(cls.restype, name="资源类型")
cls.graph.create(type)
service = Node(cls.service, name="服务")
cls.graph.create(service)
#############################################################
rel = Relationship(
cls.graph.find_one(label=cls.library), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="位置"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.building), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="别名"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.building), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="位置"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="位置"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="坐标"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="开放日"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="开放时间"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="描述"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="电话"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="借阅时间"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.room), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="借阅"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.resource), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="别名"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.resource), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="描述"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.resource), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="数量"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.restype), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="别名"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.restype), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="描述"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.restype), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="数量"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.card), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="别名"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.card), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="年龄"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.card), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="功能"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.service), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="时间"))
cls.graph.create(rel)
rel = Relationship(
cls.graph.find_one(label=cls.service), "属性",
cls.graph.find_one(label=cls.attribute,
property_key='name',
property_value="描述"))
cls.graph.create(rel)
#############################################################
rel = Relationship(cls.graph.find_one(label=cls.room), "位于",
cls.graph.find_one(label=cls.floor))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.building), "包括",
cls.graph.find_one(label=cls.library))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.room), "处于",
cls.graph.find_one(label=cls.building))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.floor), "处于",
cls.graph.find_one(label=cls.building))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.resource), "属于",
cls.graph.find_one(label=cls.restype))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.resource), "存放",
cls.graph.find_one(label=cls.room))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.room), "证件",
cls.graph.find_one(label=cls.card))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.service), "发生",
cls.graph.find_one(label=cls.room))
cls.graph.create(rel)
rel = Relationship(cls.graph.find_one(label=cls.service), "对象",
cls.graph.find_one(label=cls.resource))
cls.graph.create(rel)
def register(self, password):
if not self.find():
user = Node("User", username=self.username, password=password)
graph.create(user)
return True
return False
def CreateNode(className, lableName, name):
test_node = Node(className, lable=lableName, name=name)
graph.create(test_node)
def buildKnowledgeGraph(filename):
loadstr = ""
if filename == None:
return False
entities = {}
links = []
typedict = ['countries', 'capitals', 'currencies', 'continents']
with open(filename, encoding='UTF-8') as csvfile:
csv_reader = csv.reader(csvfile)
header = next(csv_reader)
index = 0
idlist = 0
typeEntites = {}
typeRelations = {}
for row in csv_reader:
en1 = row[0].split('/')[-1][:-1]
typeindex = -2
splitrow = row[0].split('/')
while splitrow[typeindex] not in typedict:
typeindex -= 1
type = splitrow[typeindex]
en1 = type + en1
if en1 not in entities:
entities[en1] = {'fulltext': row[0], 'nid': idlist}
idlist += 1
if 'type' not in entities[en1]:
entities[en1]['type'] = type
relation = ""
relationType = 0
if row[1][0] == '<':
relation = row[1].split('/')[-1][:-1]
if '#' in relation:
relation = relation.split('#')[-1]
else:
relation = row[1]
en2 = ""
if row[2][0] == '<':
en2 = row[2][1:-1]
if relation == "rdf:type":
en2 = en2.split("/")[-1]
if '#' in en2:
en2 = en2.split('#')[-1]
elif relation == "owl:sameAs":
if en2[-1] == '/':
en2 = en2[:-1]
en2 = en2.split("/")[-1]
elif en2.split("/")[-2] not in [
"countries", "currencies", "continents"
] and en2.split("/")[-3] not in ["capitals"]:
if en2[-1] == '/':
en2 = en2[:-1]
en2 = en2.split("/")[-1]
if '#' in en2:
en2 = en2.split('#')[-1]
else:
en2 = en2.split("/")[-1]
splitrow = row[2].split('/')
typeindex = -2
while splitrow[typeindex] not in typedict:
typeindex -= 1
type2 = splitrow[typeindex]
links.append((en1, relation, type2 + en2, row[1]))
# 未在第一列出现的实体将在处理时被删除
continue
else:
en2 = row[2]
if en1 in entities:
if relation in entities[en1]:
entities[en1][relation].append(en2)
else:
entities[en1][relation] = [en2]
else:
entities[en1] = {}
entities[en1][relation] = [en2]
entityNode = {}
nodes = []
relationships = []
for e in entities:
entity = entities[e]
#if entity['name'] == "AF#AF":
# print(entity)
#print(entity)
#print(entityNode["rdf:type"])
#if "rdf:type" in entity:
# print(True)
uppertype = entity["type"].upper()
node = Node(uppertype, fulltext=entity['fulltext'])
if uppertype in typeEntites:
if uppertype == 'COUNTRIES':
typeEntites[uppertype].append(
json.dumps({
"nid": entity['nid'],
"name": entity['name'][0],
"population": entity['population'][0]
}))
else:
typeEntites[uppertype].append(
json.dumps({
"nid": entity['nid'],
"name": entity['name'][0]
}))
else:
if uppertype == 'COUNTRIES':
typeEntites[uppertype] = [
json.dumps({
"nid": entity['nid'],
"name": entity['name'][0],
"population": entity['population'][0]
})
]
else:
typeEntites[uppertype] = [
json.dumps({
"nid": entity['nid'],
"name": entity['name'][0]
})
]
#print(entity['rdf:type'][0].upper())
#if entity['type'].upper() == 'CONTINENTS':
# print(entity)
#print(entity["rdf:type"][0].upper())
for attr in entity:
if attr == 'fulltext' or attr == 'type':
continue
node[attr] = entity[attr]
entityNode[e] = node
nodes.append(node)
#graph.create(node)
for l in links:
if l[2] in entityNode:
upperr = l[1].upper()
link = Relationship(entityNode[l[0]], upperr, entityNode[l[2]])
link['fulltext'] = l[3]
link['relation'] = l[1].upper()
relationships.append(link)
if upperr in typeRelations:
typeRelations[upperr].append(
json.dumps({
"start": entityNode[l[0]]['nid'],
"end": entityNode[l[2]]['nid']
}))
else:
typeRelations[upperr] = [
json.dumps({
"start": entityNode[l[0]]['nid'],
"end": entityNode[l[2]]['nid']
})
]
#graph.create(link)
loadstr += "{\"Labels\":{"
for label in typeEntites:
loadstr = loadstr + "\"" + label + "\"" + ": ["
for k in typeEntites[label]:
loadstr += k
loadstr += ','
loadstr = loadstr[:-1] + '],'
loadstr = loadstr[:-1] + '},'
loadstr += "\"Relationships\":{"
for label in typeRelations:
loadstr = loadstr + "\"" + label + "\"" + ": ["
for k in typeRelations[label]:
loadstr += k
loadstr += ','
loadstr = loadstr[:-1] + '],'
loadstr = loadstr[:-1] + '}}'
#print(loadstr)
#print(json.loads(loadstr))
#with open("ks.txt", 'w') as fwrite:
# fwrite.write(loadstr)
totalgraph = Subgraph(nodes=nodes, relationships=relationships)
graph.create(totalgraph)
return loadstr
def create_node(self, label, **kwargs) -> object:
n = Node(label, **kwargs)
tx = self._graph.begin(autocommit=True)
tx.create(n)
return n
def create_person(self, name):
node = Node("Person", name=name)
self.neo.create(node)
return node
def save(self, pull):
tx = self.db.begin()
# Nó da pull
unnodepull = Node("pull", **filter_props(pull))
tx.create(unnodepull)
# Nó do usuário
unnodeuser = Node("user", **filter_props(pull["user"]))
tx.merge(unnodeuser, "user", "id")
reluserpull = Relationship(unnodeuser, "CRIOU", unnodepull)
tx.create(reluserpull)
# Condicional
if ("merged_by" in pull):
# Nó do merged_by
unnodemergedby = Node("user", **filter_props(pull["merged_by"]))
tx.merge(unnodemergedby, "user", "id")
relmergedbypull = Relationship(unnodemergedby, "DEU_MERGE_EM",
unnodepull)
tx.create(relmergedbypull)
# Condicional
if (pull["milestone"]):
# nó da milestone
unnodemilestone = Node("milestone",
**filter_props(pull["milestone"]))
tx.merge(unnodemilestone, "milestone", "id")
relmilestonepull = Relationship(unnodemilestone, "CONTÉM",
unnodepull)
tx.create(relmilestonepull)
# nó do criador do milestone
unnodemilestonecreator = Node(
"user", **filter_props(pull["milestone"]["creator"]))
tx.merge(unnodemilestonecreator, "user", "id")
relcreatormilestone = Relationship(unnodemilestonecreator, "CRIOU",
unnodemilestone)
tx.create(relcreatormilestone)
# Nós das labels
lsnodelabel = [
Node("label", **filter_props(label)) for label in pull['labels']
]
for nodelabel in lsnodelabel:
tx.merge(nodelabel, "label", "id")
rellabelpull = Relationship(nodelabel, "ROTULA", unnodepull)
tx.create(rellabelpull)
# Nós dos assignees
lsnodeassignee = [
Node("user", **filter_props(assignee))
for assignee in pull["assignees"]
]
for nodeassignee in lsnodeassignee:
tx.merge(nodeassignee, "user", "id")
relassigneepull = Relationship(nodeassignee, "ADMINISTRA",
unnodepull)
tx.create(relassigneepull)
# Nós dos reviewers
lsnodereviewers = [
Node("user", **filter_props(reviewer))
for reviewer in pull["requested_reviewers"]
]
for nodereviewer in lsnodereviewers:
tx.merge(nodereviewer, "user", "id")
relreviewerissue = Relationship(nodereviewer,
"CHAMADO_PARA_REVISAR", unnodepull)
tx.create(relreviewerissue)
# Nós dos teams
lsnodeteams = [
Node("team", **filter_props(team))
for team in pull["requested_teams"]
]
for nodeteam in lsnodeteams:
tx.merge(nodeteam, "team", "id")
relteamissue = Relationship(nodeteam, "CHAMADO_PARA_REVISAR",
unnodepull)
tx.create(relteamissue)
# Concluir
tx.commit()
for one in os.listdir(binary_path):
file_list.append(one)
file_attr_list.extend(obtain_sym_table(binary_path + one, "FILE"))
func_attr_list.extend(obtain_sym_table(binary_path + one, "FUNC"))
file_arch_list.append(get_file_arch(binary_path + one))
file_attr_list = list(set(file_attr_list))
func_attr_list = list(set(func_attr_list))
file_node_list = []
file_attr_node_list = []
func_attr_node_list = []
for index, one in enumerate(file_list):
arch = file_arch_list[index]
file_node_list.append(Node(arch, name=one))
for one in file_attr_list:
file_attr_node_list.append(Node("FILE table", name=one[1]))
for one in func_attr_list:
func_attr_node_list.append(Node("FUNC table", name=one[1]))
print("Last %d items in file_attr_list" % len(file_attr_list))
print("Last %d items in func_attr_list" % len(func_attr_list))
file_count = 0
for index, one in enumerate(file_list):
file_count += 1
file_table = obtain_sym_table(binary_path + one, "FILE")
'gender': 'male'
}, {
'id': '20170101',
'name': 'Jordan',
'age': 20,
'gender': 'male'
}, {
'id': '20170101',
'name': 'Jordan',
'age': 20,
'gender': 'male'
}
global info_id, info_name, info_age, info_gender, node
for td in test_data:
print(td.keys())
for tk in td.keys():
if 'id' in tk:
info_id = tk
if 'name' in tk:
info_name = tk
if 'age' in tk:
info_age = tk
if 'gender' in tk:
info_gender = tk
node = Node(id=info_id, name=info_name, age=info_age, gender=info_gender)
print('result:' + info_id, ':', info_name, ':', info_age, ':', info_gender)
print(node)
graph.create(node)
def data_service_nodes(dataserver):
data_services = [Node("data_access_service",name=dataserver,service_type="http", status='up'),
Node("data_access_service",name=dataserver,service_type="opendap",status='up'),
Node("data_access_service",name=dataserver,service_type="globus",status='up')
]
return data_services
def create_node(cls, workbook):
"""建立馆室节点"""
room_sheet = workbook.sheet_by_index(0)
for i in range(1, room_sheet.nrows - 6):
row = room_sheet.row_values(i)
name = row[0]
if row[0].find("_") != -1:
name_arr = row[0].split("_")
if len(name_arr) >= 3:
name = name_arr[2]
room_node = Node(cls.room,
type=cls.room,
name=name,
office_name=row[0],
variant_name=row[1],
position=row[2],
describe=row[3],
open_date=row[4],
phone=row[5],
monday_open=row[6],
monday_borrow=row[7],
tuseday_open=row[8],
tuseday_borrow=row[9],
wednesday_open=row[10],
wednesday_borrow=row[11],
thursday_open=row[12],
thursday_borrow=row[13],
friday_open=row[14],
friday_borrow=row[15],
saturday_open=row[16],
saturday_borrow=row[17],
sunday_open=row[18],
sunday_borrow=row[19],
area=row[20],
floor=row[21],
card=row[22],
des_x=row[25],
des_y=row[26],
borrow=row[29])
cls.graph.create(room_node)
for i in range(room_sheet.nrows - 6, room_sheet.nrows):
row = room_sheet.row_values(i)
mark_node = Node(cls.mark,
type=cls.mark,
name=row[0],
office_name=row[0],
self_site=row[24])
cls.graph.create(mark_node)
"""建立馆区节点"""
building_sheet = workbook.sheet_by_index(2)
for i in range(1, building_sheet.nrows):
row = building_sheet.row_values(i)
name = row[0]
#print(name)
building_node = Node(cls.building,
type=cls.building,
name=name,
office_name=row[0],
variant_name=row[1],
position=row[2],
workdate=row[3],
weekdate=row[4],
worktime=row[5],
weektime=row[6])
cls.graph.create(building_node)
"""建立楼层节点"""
floor_sheet = workbook.sheet_by_index(4)
for i in range(1, floor_sheet.nrows):
row = floor_sheet.row_values(i)
name = row[0]
#print(name)
floor_node = Node(cls.floor,
type=cls.floor,
name=name,
office_name=row[0],
variant_name=row[1],
area=row[2],
upstair=row[3],
downstair=row[4])
cls.graph.create(floor_node)
"""建立资源节点"""
resource_sheet = workbook.sheet_by_index(3)
for i in range(1, resource_sheet.nrows):
row = resource_sheet.row_values(i)
#print(row)
name = row[0]
if row[0].find("_") != -1:
name = row[0].split("_")[1]
# print(name)
resource_node = Node(cls.resource,
type=cls.resource,
name=name,
office_name=row[0],
variant_name=row[1],
describe=row[2],
count=row[3],
room=row[4],
belong=row[5],
ctime=row[6],
range=row[8],
topic=row[9])
cls.graph.create(resource_node)
'''建立证件节点'''
card_sheet = workbook.sheet_by_index(1)
for i in range(1, card_sheet.nrows):
row = card_sheet.row_values(i)
card_node = Node(cls.card,
type=cls.card,
name=row[0],
office_name=row[0],
variant_name=row[1],
age=row[2],
function=row[3])
cls.graph.create(card_node)
'''建立服务节点'''
service_sheet = workbook.sheet_by_index(7)
for i in range(1, service_sheet.nrows):
row = service_sheet.row_values(i)
service_node = Node(cls.service,
type=cls.service,
name=row[0],
office_name=row[0],
variant_name=row[1],
date=row[4],
time=row[5],
discribe=row[6],
card=row[7])
cls.graph.create(service_node)
'''建立资源类型节点'''
restype_sheet = workbook.sheet_by_index(6)
for i in range(1, restype_sheet.nrows):
row = restype_sheet.row_values(i)
restype_node = Node(cls.restype,
type=cls.restype,
name=row[0],
office_name=row[0],
variant_name=row[1],
describe=row[2],
count=row[3],
room=row[4],
belong=row[5])
cls.graph.create(restype_node)
'''建立国家图书馆节点'''
variant_name = '图书馆，国家图书馆'
describe = '国家图书馆的前身是建于1909年（清宣统元年）9月9日的京师图书馆，1912年8月27日开馆接待读者，1916年京师图书馆按规定正式接受国内出版物呈缴本，开始履行国家图书馆的部分职能，1928年7月更名为国立北平图书馆，1929年8月与北平北海图书馆合并，仍名国立北平图书馆， 1950年3月6日国立北平图书馆更名为国立北京图书馆，1951年6月12日更名为北京图书馆，1998年12月12日经国务院批准，北京图书馆更名为国家图书馆，对外称中国国家图书馆。包括文津街古籍馆、白石桥总馆南区和总馆北区三个馆区。'
library = Node(cls.library,
type=cls.library,
name=cls.library,
office_name=cls.library,
variant_name=variant_name,
describe=describe)
cls.graph.create(library)
'''建立资源大类'''
res_sheet = workbook.sheet_by_index(8)
for i in range(1, res_sheet.nrows):
row = res_sheet.row_values(i)
res_node = Node(cls.res,
type=cls.res,
name=row[0],
office_name=row[0],
variant_name=row[0],
num=row[1])
cls.graph.create(res_node)
from py2neo import Graph
graph = Graph(password="******")
from py2neo import Node, Relationship
alice = Node("Person", name="Alice")
bob = Node("Person", name="Bob")
alice_knows_bob = Relationship(alice, "KNOWS", bob)
graph.create(alice_knows_bob)
messages_proxy.append(j['event']['event_id'])
count+=1
print "\n#####"+str(count)+"#####"
messages_1.append(j)
graph = Graph(password = '******', host = 'localhost')
graph.delete_all()
count = 0
for message in messages_1:
try:
new_node = Node('Event',
name = message['event']['event_name'],
id = message['event']['event_id'],
time = message['event']['time'],
url = message['event']['event_url'],
venue_name = ['venue']['venue_name'],
long = ['venue']['lon'],
lat = ['venue']['lat'],
venue_id = ['venue']['id'])
graph.create(new_node)
except:
new_node = Node('Event',
name = message['event']['event_name'],
id = message['event']['event_id'],
url = message['event']['event_url'])
graph.create(new_node)
count+=1
print"##node number\t"+str(count)+"\t just created##"
print(len(messages),len(messages_1),len(messages_proxy))
def test_graph_contains(graph):
node = Node()
graph.create(node)
assert node.graph is graph
graph = Graph(password="******")
sentences=['Qasim hates peer',
'Malaika is daughter of Ali',
'Hussain loves Cat',
'Bashir has a Dog',
'Cat sits in Car'
]
for sentence in sentences:
tokens=word_tokenize(sentence)
print(tokens)
tags= pos_tag(tokens)
print(tags)
nouns=[]
verb=''
for tuple in tags:
print(tuple)
if tuple[1]=='NNP':
nouns.append(tuple[0])
if tuple[1]=='VBZ' or tuple[1]=='NN':
verb=tuple[0]
node1 = Node("Person", name=nouns[0])
graph.create(node1)
node2 = Node("Object", name = nouns[1])
graph.create(node2)
relationship = Relationship(node1, verb, node2)
graph.create(relationship)
