def database_connection_test(prints=False):
if prints:
print '=' * 100
py2neo.authenticate(HOST, USER, PASS)
try:
graph = py2neo.Graph(GRAPH)
if graph:
if prints:
print "databases is working..."
return graph
except Exception as err:
print unidecode.unidecode(unicode(err.message))
try:
print "trying to execute:", neo4j_path % "start"
subprocess.call(neo4j_path % "start", shell=True)
for x in range(0, 11):
bar = '=' * x
print '\rwaiting for server delay', '[', bar.ljust(10), ']',
time.sleep(1)
graph = py2neo.Graph(GRAPH)
return graph
except Exception as err:
print "subprocess ERROR:", unidecode.unidecode(unicode(err))
raise err
finally:
if prints:
print '=' * 100
def get_graph():
if config.NEO4J_USER is not None:
return py2neo.Graph(
config.NEO4J_URL, password=config.NEO4J_PASSWORD, user=config.NEO4J_USER
)
else:
return py2neo.Graph(config.NEO4J_URL)
def __init__(self, database_url=None, models=[]):
"""
Initializes the graph with the models that make up the schema graph and
an identifier for a url to a neo4j database.
"""
self.neograph = py2neo.Graph() if not database_url else py2neo.Graph(
database_url)
self.models_dict = {}
for model in models:
self.models_dict[model.__name__] = model
def get_graph(graph_name):
if graph_name == "marclab":
neo.authenticate('localhost:7474', 'neo4j', 'neo4j')
neo_graph = neo.Graph('http://localhost:7474/db/data')
elif graph_name == "movies":
neo.authenticate('localhost:7475', 'neo4j', 'neo4j')
neo_graph = neo.Graph('http://localhost:7475/db/data')
elif graph_name == "flights":
neo.authenticate('localhost:7476', 'neo4j', 'neo')
neo_graph = neo.Graph('http://localhost:7476/db/data')
return neo_graph
def __init__(self, *a, concurrent_queries: int = 1, **k):
self.executor = ThreadPoolExecutor(max_workers=concurrent_queries)
self.graph = py2neo.Graph(*a, **k)
self.graphs = Queue()
for _ in range(concurrent_queries):
self.graphs.put(py2neo.Graph(*a, **k))
self.__dict__.update(
{parent_attribute: getattr(self.graph, parent_attribute)
for parent_attribute in dir(self.graph)
if not parent_attribute.startswith('_')
and parent_attribute not in ['data', 'run', 'evaluate']}
)
def get_graph(self):
if "GC_NEO4J_URL" in os.environ:
url = os.getenv("GC_NEO4J_URL")
if "GC_NEO4J_USER" in os.environ and "GC_NEO4J_PASSWORD" in os.environ:
user = os.getenv("GC_NEO4J_USER")
pw = os.getenv("GC_NEO4J_PASSWORD")
print("URL", url)
print("pw", pw)
print("user", user)
return py2neo.Graph(url, password=pw, user=user)
return py2neo.Graph(url)
else:
return py2neo.Graph(self.NEO4J_CON)
def test_neo_many_node(self):
self._clear_tables()
count = 200
test_props = []
with self.psqlDriver.session_scope():
for i in range(count):
node_id = str(uuid.uuid4())
timestamp = long(time.time())
props = {'timestamp': timestamp}
test_props.append({
'id': node_id, 'timestamp': (timestamp)
})
self.psqlDriver.node_merge(node_id, label='test', properties=props)
self.driver.export(self.get_csv_dir())
self.batch_import()
self.neo4jDriver = py2neo.Graph()
nodes = self.neo4jDriver.cypher.execute('match (n:test) return n')
self.assertEqual(len(nodes), count)
node_props = [n.n.properties for n in nodes]
for prop in test_props:
self.assertTrue(prop in node_props)
for node_prop in node_props:
self.assertTrue(node_prop in test_props)
def py2neo_view(request):
"""
py2neo (community driver)
Pass through CQL query and return JSON response
"""
user, password = get_graphenedb_credentials()
cqlQuery = request.body
if not cqlQuery:
return Response({'detail': 'Empty query'}, status=400)
py2neo.authenticate(REST_ENDPOINT, user, password)
graph = py2neo.Graph(BOLT_ENDPOINT,
user=user,
password=password,
bolt=True,
secure=True,
http_port=24789,
https_port=24780)
try:
results = graph.data(cqlQuery)
return Response(results)
except Exception as e:
return Response({'detail': unicode(e)}, status=400)
def do_POST(self):
graph = py2neo.Graph(neo4jLoc)
parsedUrl = urlparse.urlparse(self.path)
splitPath = parsedUrl.path.lstrip("/").split("/")
if (splitPath[0] == "usersToOrgs"):
orgData = urlparse.parse_qs(
self.rfile.read(int(
self.headers.get('content-length'))).decode('ascii'))
setOrgs(
graph, orgData,
urllib.parse.unquote(parsedUrl.query).replace(
"fileName=", "", 1))
elif (splitPath[0] == "load"):
filename = self.getFilePathFromPostData()
with open(filename) as file:
if "url" in filename and "project" in filename:
populateNeoDb(graph, file.read(), "jira")
elif "owner" in filename and "repository" in filename:
populateNeoDb(graph, file.read(), "github")
elif "hostname" in filename:
populateNeoDb(graph, file.read(), "gerrit")
elif "uri" in filename:
populateNeoDb(graph, file.read(), "email")
elif (splitPath[0] == "deleteData"):
os.remove(self.getFilePathFromPostData())
elif (splitPath[0] == "reset"):
clearGraph(graph)
elif (splitPath[0] == "merge"):
print("merge")
userKeys = urlparse.parse_qs(
self.rfile.read(int(
self.headers.get('content-length'))).decode('ascii'))
mergeUsers(graph, userKeys)
def authenticateNeo4j(host, user, password):
try:
Graph = py2neo.Graph(host=host, user=user, password=password)
except:
writeToFile("temp.html", "Unable to access neo4j")
exit()
return Graph
def drawRelations(p1,v1,p2,v2,r,directed = False):
graph = py2neo.Graph("http://localhost:7474",auth=("neo4j","961121"))
graph.run("MATCH(e),(f)\
WHERE e.%s = '%s' AND f.%s = '%s'\
MERGE(e)-[r:%s]->(f)\
RETURN r"%(p1,v1,p2,v2,r))
def push(self, **kwargs):
# send data to Invoker
# out: (1) 如果数据库没有此实体，请返回 {}
# (2) 如果有数据，返回格式：
#self.result = {
# 'entity': [ {
# name: '名称',
# des: '详细简略信息，<字数控制在60字左右>',
# symbolSize: 60, # 查询实体大小为60, 其余取30
# itemStyle: {
# normal: {
# color: 'red' # 环状颜色填充：相同类型相同颜色，不同类型不同颜色['#f5a5c0', 'yellow', '#f4c89e', '#c23531', '#f2acf6', '#acf3f6', 'acf6c9']
# }
# }
# },...],
# 'relation':
# [{
# source: '蔡成功', # entity1
# target: '欧阳菁', # entity2
# name: "举报" # 关系名
# }...]
# }
self.result={'entity':'李四'}
entity = kwargs.get('entity')
print(entity)
grath = neo.Graph('http://localhost:7474',username='******',password='******')
return self.result
def test_neo4j_remote(self, mock_get):
test_context = 'PYBEL_TEST_CTX'
neo_path = os.environ['NEO_PATH']
try:
neo = py2neo.Graph(neo_path)
neo.data(
'match (n)-[r]->() where r.{}="{}" detach delete n'.format(
PYBEL_CONTEXT_TAG, test_context))
except py2neo.database.status.GraphError:
self.skipTest("Can't query Neo4J ")
except:
self.skipTest("Can't connect to Neo4J server")
else:
with self.runner.isolated_filesystem():
args = [
'convert', '--path', test_bel_simple, '--connection',
self.connection, '--neo', neo_path, '--neo-context',
test_context
]
self.runner.invoke(cli.main, args)
q = 'match (n)-[r]->() where r.{}="{}" return count(n) as count'.format(
PYBEL_CONTEXT_TAG, test_context)
count = neo.data(q)[0]['count']
self.assertEqual(14, count)
def _connect(self):
if not self.__graph:
self.__graph = py2neo.Graph(sceme='http',
host=self.__host,
http_port=int(self.__port),
user=self.__user,
password=self.__password)
def delete_data_set_neo4j(dataset_uuid):
"""Remove a dataset's related node in Neo4J.
"""
logger.debug('Deleted data set (uuid: %s) in Neo4J', dataset_uuid)
graph = py2neo.Graph(urljoin(settings.NEO4J_BASE_URL, 'db/data'))
statement = (
"MATCH (ds:DataSet {uuid:{dataset_uuid}}) "
"OPTIONAL MATCH (ds)-[r]-() "
"DELETE ds, r"
)
try:
graph.cypher.execute(
statement,
parameters={
'dataset_uuid': dataset_uuid
}
)
except Exception as e:
""" Cypher queries are expected to fail and raise an exception when
Neo4J is not running or when transactional queries are not available
(e.g. Travis CI doesn't support transactional queries yet)
"""
logger.error(
'Failed to remove dataset (uuid: %s) from Neo4J. '
'Exception: %s', dataset_uuid, e
)
def connectToDB(host="localhost",
port=7687,
user="******",
password="******"):
try:
driver = py2neo.Graph(host=host,
port=port,
user=user,
password=password)
except py2neo.database.DatabaseError as err:
raise py2neo.database.DatabaseError(
"Database failed to service the request. {}".format(err))
except py2neo.database.ClientError as err:
raise py2neo.ClientError(
"The client sent a bad request. {}".format(err))
except py2neo.database.TransientError as err:
raise py2neo.TransientError(
"Database cannot service the request right now. {}".format(err))
except py2neo.GraphError as err:
raise py2neo.GraphError("{}".format(err))
except Exception as err:
exc_type, exc_obj, exc_tb = sys.exc_info()
fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
sys_error = "{}, file: {},line: {}".format(sys.exc_info(), fname,
exc_tb.tb_lineno)
raise Exception("Unexpected error:{}.\n{}".format(err, sys_error))
return driver
def connect(config):
RetryConfig.configure(config)
env_keys_map = {
'host': 'neo4host',
'login': '******',
'password': '******'
}
config_keys_map = {'login': '******', 'password': '******'}
config_section = 'neo4j'
args = {}
for option in ('host', 'login', 'password'):
name = env_keys_map.get(option, option)
value = os.environ.get(name)
if not value:
name = config_keys_map.get(option, option)
value = config.get(config_section, name)
args[option] = value
uri_format = "bolt://{login}:{password}@{host}:7687".format
uri = uri_format(**args)
args['password'] = '******' * 5
logger.info('NEO4j connect %s', uri_format(**args))
return py2neo.Graph(uri)
def test_neo_star_topology(self):
"""
Create a star topology, verify lookup by src_id and that all nodes
are attached
"""
self._clear_tables()
leaf_count = 10
src_id = str(uuid.uuid4())
with self.psqlDriver.session_scope():
self.psqlDriver.node_merge(node_id=src_id, label='test')
dst_ids = [str(uuid.uuid4()) for i in range(leaf_count)]
for dst_id in dst_ids:
self.psqlDriver.node_merge(node_id=dst_id, label='test')
self.psqlDriver.edge_insert(PolyEdge(
src_id=src_id, dst_id=dst_id, label='edge1'))
self.driver.export(self.get_csv_dir())
self.batch_import()
self.neo4jDriver = py2neo.Graph()
nodes = self.neo4jDriver.cypher.execute(
'match (n)-[r]-(m) where n.id = "{src_id}" return n'.format(
src_id=src_id)
)
self.assertEqual(len(nodes), leaf_count)
def extract_data_and_create_node(data):
kg = py2neo.Graph("localhost:7474", username="******", password="******")
name_ls = []
faction_ls = []
monarch_ls = []
for row in data:
row = json.loads("".join(i for i in row))
name_ls.append(row["name"])
if type(row["faction"]) is not list:
if row["faction"] is not None:
faction_ls.append(row["faction"])
monarch_ls.append(row["monarch"])
for i in faction_ls:
if "," in i:
faction_ls += i.split(",")
faction_ls.remove(i)
name_ls = list(set(name_ls))
faction_ls = list(set(faction_ls))
# create_node("people",name_ls,kg)
# create_node("faction",faction_ls,kg)
# create_people_property(data,kg)
create_monarch_rel(data, kg)
create_faction_rel(data, kg)
def on_btnConfirms_clicked(self):
# self.ui.table_sql.setVisible(True)
# self.url='http://www.baidu.com'
# self.url = 'file:///' + os.path.dirname(os.path.dirname(d)) + \
# '/static/html/show.html'
# self.url = self.url.replace('\\', '/')
# self.browser.page().load(QUrl(self.url))
# self.browser.show()
self.browser.setVisible(True)
self.ui.table_sql.setVisible(True)
self.browser_kg.setVisible(False)
search_name = self.ui.Check_lineEdit.text()
graph = py2neo.Graph("http://localhost:7474", auth=("neo4j", "961121"))
str1 = "MATCH(n{name:'%s'})RETURN n" % (search_name)
result = graph.run(str1).to_subgraph()
# print(len(result.nodes))
if result == None:
return
if len(result.nodes) > 1:
return
# print(result.identity)
search_name = result.identity
# a = str(result) # f**k neo4j
# neodict=dict(result) if result else None
self.neo = search.search_return(search_name)
node_dict = self.neo.dict()
print(node_dict)
if node_dict:
self.__createSearch(node_dict)
# self.__draw(self.neo.picture_path())
self.ToJS_showNode.emit(self.neo.singleNode_json())
def main():
dup_doc_files = glob.glob("/Users/zys/project/Doc_dup_test.csv")
dup_sen_files = glob.glob("/Users/zys/project/Sen_dup_test.csv")
cv_files = glob.glob("/Users/zys/project/cv_doc_test.csv")
db_graph = pn.Graph()
res = {}
cvterms = load_cv_doc_batch(cv_files[:3])
# for w in [0.1,0.3,0.4,0.5,0.6,0.7,0.8,0.9]:
for w in [0.5]:
start = time.time()
G = load_graph_batch(dup_doc_files[:3], dup_sen_files[:3], 1, 5,
cvterms)
print("--- building : %s seconds ---" % (time.time() - start))
start = time.time()
best_par = get_partition_by_cc(G, db_graph, w)
print("--- event : %s seconds ---" % (time.time() - start))
outfile = "/Users/zys/project/res_test_" + str(w) + "_5.txt"
out_doc_file = "/Users/zys/project/res_doc_test" + str(w) + ".txt"
textlist = get_text_list(best_par) # for doc level type (docid, text)
doclist = get_doc_list(best_par)
res['keywords_set'] = textlist
res['doc_set'] = doclist
out_to_file(outfile, best_par)
out_to_file(out_doc_file, res['doc_set'])
def convert(path, connection, csv, sif, gsea, graphml, json, pickle, bel, neo,
neo_context, store, allow_naked_names, allow_nested,
allow_unqualified_translocations, no_identifier_validation,
no_citation_clearing, required_annotations):
"""Convert BEL."""
manager = Manager(connection=connection)
g = from_lines(
path,
manager=manager,
allow_nested=allow_nested,
allow_naked_names=allow_naked_names,
allow_unqualified_translocations=allow_unqualified_translocations,
citation_clearing=(not no_citation_clearing),
required_annotations=required_annotations,
no_identifier_validation=no_identifier_validation,
use_tqdm=True,
)
if csv:
log.info('Outputting CSV to %s', csv)
to_csv(g, csv)
if sif:
log.info('Outputting SIF to %s', sif)
to_sif(g, sif)
if graphml:
log.info('Outputting GraphML to %s', graphml)
to_graphml(g, graphml)
if gsea:
log.info('Outputting GRP to %s', gsea)
to_gsea(g, gsea)
if json:
log.info('Outputting JSON to %s', json)
to_json_file(g, json)
if pickle:
log.info('Outputting pickle to %s', pickle)
to_pickle(g, pickle)
if bel:
log.info('Outputting BEL to %s', bel)
to_bel(g, bel)
if store:
log.info('Storing to database')
to_database(g, manager=manager, store_parts=True)
if neo:
import py2neo
log.info('Uploading to neo4j with context %s', neo_context)
neo_graph = py2neo.Graph(neo)
assert neo_graph.data(
'match (n) return count(n) as count')[0]['count'] is not None
to_neo4j(g, neo_graph, neo_context)
sys.exit(0 if 0 == len(g.warnings) else 1)
def extract_from_neo4j(csv_path, index, neo4j_uri, username, password):
'''
从neo4j中提取数据
:param csv_path:
:param index:
:param neo4j_uri:
:param username:
:param password:
:return:
'''
print(f'进程{os.getpid()}：正在查询第{index}层企业节点')
s = time.time()
wf = open(csv_path, 'w', encoding='utf8')
if index != 9:
command = 'match (n:GS) -[:IPEE|:BEE* %s .. %s]-> (m:GS) where not (:GS) -[:IPEE|:BEE]-> (n) return distinct m.ID as nid'
else:
command = 'match (n:GS) -[:IPEE|:BEE* %s .. %s]-> (m:GS) return distinct m.ID as nid'
graph = py2neo.Graph(uri=neo4j_uri, username=username, password=password)
ret = graph.run(command % (index, index)).data()
count = 0
for i in ret:
count += 1
wf.write(','.join([i['nid'], str(index), '\n']))
wf.close()
print(f'{index}层企业查询完毕: {count},耗时{time.time()-s}秒！')
return None
def remove_read_access_in_neo4j(dataset_uuids, user_ids):
"""Remove read access for one or multiple users to one or more datasets.
"""
logger.debug(
'Removing read access from users (%s) to data set (uuid: %s) in Neo4J',
user_ids, dataset_uuids)
graph = py2neo.Graph(urljoin(settings.NEO4J_BASE_URL, 'db/data'))
statement = (
"MATCH (ds:DataSet {uuid:{dataset_uuid}}), (u:User {id:{user_id}}) "
"MATCH (ds)<-[r:`read_access`]-(u) "
"DELETE r")
try:
tx = graph.cypher.begin()
for dataset_uuid in dataset_uuids:
for user_id in user_ids:
tx.append(statement, {
'dataset_uuid': dataset_uuid,
'user_id': user_id
})
tx.commit()
except Exception as e:
""" Cypher queries are expected to fail and raise an exception when
Neo4J is not running or when transactional queries are not available
(e.g. Travis CI doesn't support transactional queries yet)
"""
logger.error(
'Failed to remove read access from users (%s) for datasets '
'(uuids: %s) from Neo4J. Exception: %s', user_ids, dataset_uuids,
e)
def __init__(self, label):
self.label = label
self.graph = py2neo.Graph(neo4j_url)
self.node_by_name = {}
self.other_nodes = {}
self._query = "match (n:{label}) where n.name =~ '{name}.*' return n"
self._query_to_other = "match (a:{label})--(z:{other}) where id(a) = {a} return z"
def load_food_graph(authentication, host, port, protocol):
"""
Loads the food graph database from Neo4J.
Parameters:
authentication: Two-par credentials tuple (username and password) used to authenticate to
the Neo4J food database.
host: Name of the host server where the Neo4J database is active.
port: Communication port of the Neo4J database on the host server.
protocol: Application-layer protocol used to communicate with the Neo4J database.
"""
logger.debug('Loading food graph DB')
gr = py2neo.Graph(auth=authentication,
host=host,
port=port,
scheme=protocol)
# Verify if the DB is active
try:
dbname = gr.database.name
except ServiceUnavailable:
# Log the error and show traceback.
logger.error('Neo4J service not available. Is it running?')
tb.print_last()
# Exit the application with error code.
sys.exit(errno.ECONNREFUSED)
else:
return gr
def social_graph(settings):
try:
py2neo.authenticate(settings['NEO4J_HOST'], settings['NEO4J_USER'],
settings['NEO4J_PASS'])
return py2neo.Graph(settings['NEO4J_CONNECTION_URI'])
except Exception as ex:
current_app.logger.exception("Exception At init_sc_app: ", ex)
raise
def __init__(self):
self.g = py2neo.Graph(
host="sumsky.top",
http_port=7474,
user="******", # Here should be your Neo4j username
password="******" # Here should be your Neo4j password
)
self.num_limit = 7
def startup(self):
self.graph_db = py2neo.Graph()
self.selectors = {}
# setup selectors
for node_type in NodeTypes:
self.selectors[node_type] = py2neo.NodeSelector(
self.graph_db).select(node_type.name)
self._has_started_up = True
def get_db():
if 'db ' not in g:
g.db = py2neo.Graph(
current_app.config["DATABASE"],
password=current_app.config["DB_PASSWORD"]
)
return g.db
