def __init_graph(self):
config = ConfigParser.ConfigParser()
conf_file = os.path.join(self.conf, 'neo4j.conf')
if not os.path.exists(conf_file):
conf_file = os.path.join(self.conf, 'neo4j.conf.default')
if not os.path.exists(conf_file):
log.critical("Neo4j not initialized (configuration file not found)")
self.running = False
return
config.read(conf_file)
try:
section = config.options('neo4j') #pylint:disable=unused-variable
except: #pylint:disable=bare-except
log.critical("Neo4j configuration file lacks neo4j section")
self.running = False
return
neo4j.authenticate(config.get('neo4j', 'host'),
config.get('neo4j', 'username'),
config.get('neo4j', 'password'))
self.graph = neo4j.Graph(config.get('neo4j', 'url'))
def initenviron(logname, maxdrones, debug=False, timeout=90, nanodebug=0, cmadebug=0):
'Initialize the test environment.'
logwatch = LogWatcher(logname, [], timeout, returnonlymatch=True, debug=debug)
logwatch.setwatch()
sysenv = SystemTestEnvironment(logname, maxdrones, nanodebug=nanodebug, cmadebug=cmadebug)
CMAinit(None, host=str(sysenv.cma.ipaddr), readonly=True,
neologin=SystemTestEnvironment.NEO4JLOGIN, neopass=SystemTestEnvironment.NEO4JPASS)
url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)
print >> sys.stderr, 'OPENING Neo4j at URL %s' % url
neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr,
SystemTestEnvironment.NEO4JLOGIN,
SystemTestEnvironment.NEO4JPASS)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
logger('$(grep MemFree: /proc/meminfo)', hardquote=False)
tq = QueryTest(store
, '''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone'''
, GN.nodeconstructor, debug=debug)
if not tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1
, delay=0.5, maxtries=20):
sysenv.cma.cleanupwhendone = False
raise RuntimeError('Query of "up" status failed. Weirdness')
return sysenv, store
def connection():
if hasattr(connection, 'db'):
return connection.db
url = DATABASE_URL
u = urlparse(url)
if u.netloc.find('@') > -1:
credentials, host = u.netloc.split('@')
user, password, = credentials.split(':')
neo4j.authenticate(host, user, password)
url = ''.join([u.scheme, '://', host, u.path, u.query])
try:
connection.db = neo4j.GraphDatabaseService(url)
except SocketError as e:
raise SocketError("Error connecting to {0} - {1}".format(url, e))
if connection.db.neo4j_version >= (2, 0):
raise Exception(
"Support for neo4j 2.0 is in progress but not supported by this release."
)
if connection.db.neo4j_version < (1, 8):
raise Exception("Versions of neo4j prior to 1.8 are unsupported.")
return connection.db
def __init__(self):
'''
initializes:
1. graph database connection
2. datastore connection
3. graph database indices required
4. url and templates for interaction with the graph database REST API
'''
Util.__init__(self)
if os.environ.get('NEO4J_URL'):
graph_db_url = urlparse(os.environ.get('NEO4J_URL'))
neo4j.authenticate(
"{host}:{port}".format(host = graph_db_url.hostname, port = graph_db_url.port),
graph_db_url.username, graph_db_url.password
)
self.graphdb = neo4j.GraphDatabaseService(
'http://{host}:{port}/db/data'.format(host = graph_db_url.hostname, port = graph_db_url.port)
)
else:
self.graphdb = neo4j.GraphDatabaseService()
self.node_index = self.graphdb.get_or_create_index(neo4j.Node, 'NODE')
self.disambiguation_index = self.graphdb.get_or_create_index(neo4j.Node, self.DISAMBIGUATION)
self._url = lambda present_node: 'http://localhost:7474/db/data/node/{0}'.format(present_node)
self._template = lambda target_node: {
"to" : self._url(target_node),
"relationships": {
"type": "sibling"
},
"cost_property": "weight",
"algorithm" : "dijkstra"
}
self.DataM = RelDataStore()
def get_graph():
# Connect to graph
u = urlparse(session['neo4j_url'])
neo4j.authenticate('%s:%s' % (u.hostname, u.port), 'neo4j', session['neo4j_password'])
graph = neo4j.Graph("%s/db/data/" % session['neo4j_url'])
graph.cypher.execute('match (t:Tweet) return COUNT(t)')
return graph
def get_graph():
global NEO4J_URL,NEO4J_HOST_PORT,NEO4J_USER,NEO4J_PASSWORD
# Connect to graph
neo4j.authenticate(NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD)
graph = neo4j.Graph(NEO4J_URL)
graph.cypher.execute('match (t:Tweet) return COUNT(t)')
return graph
def get_graph():
global NEO4J_URL, NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD
# Connect to graph
neo4j.authenticate(NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD)
graph = neo4j.Graph(NEO4J_URL)
graph.cypher.execute('match (t:Tweet) return COUNT(t)')
return graph
def connect():
try:
# graph_db = neo4j.GraphDatabaseService("http://localhost:7474/db/data/")
neo4j.authenticate("jobscope.sb01.stations.graphenedb.com:24789", "JobScope", "0W07c5PCLYr4yxPDd9ir")
graph_db = neo4j.GraphDatabaseService("http://jobscope.sb01.stations.graphenedb.com:24789/db/data/")
except rest.ResourceNotFound:
print "Database service not found"
return graph_db
def connect(username, password):
url = 'http://localhost:7474'
neo4j.authenticate(url.strip('http://'), username, password)
NeoGraph= Graph(url + '/db/data/')
if NeoGraph is None:
print "unconnect"
return 0
else:
print "connected with neo4j"
return NeoGraph
def __init__(self):
if os.environ.get('NEO4J_URL'):
graph_db_url = urlparse(os.environ.get('NEO4J_URL'))
# set up authentication parameters
neo4j.authenticate("{host}:{port}".format(host=graph_db_url.hostname, port=graph_db_url.port), graph_db_url.username, graph_db_url.password)
# connect to authenticated graph database
self.graph_db = neo4j.GraphDatabaseService('http://{host}:{port}/db/data'.format(host=graph_db_url.hostname, port=graph_db_url.port))
else:
self.graph_db = neo4j.GraphDatabaseService('http://localhost:7474/db/data')
def initenviron(logname,
maxdrones,
mgmtsystem,
debug=False,
cmaimage='',
nanoimages=[],
timeout=90,
nanodebug=0,
cmadebug=0):
'Initialize the test environment.'
logwatch = LogWatcher(logname, [],
timeout,
returnonlymatch=True,
debug=debug)
logwatch.setwatch()
sysenv = SystemTestEnvironment(logname,
maxdrones,
mgmtsystem,
cmaimage=cmaimage,
nanoimages=nanoimages,
nanodebug=nanodebug,
cmadebug=cmadebug)
CMAinit(None,
host=str(sysenv.cma.ipaddr),
readonly=True,
neologin=SystemTestEnvironment.NEO4JLOGIN,
neopass=SystemTestEnvironment.NEO4JPASS)
url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)
print >> sys.stderr, 'OPENING Neo4j at URL %s' % url
neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr,
SystemTestEnvironment.NEO4JLOGIN,
SystemTestEnvironment.NEO4JPASS)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
logger('$(grep MemFree: /proc/meminfo)', hardquote=False)
tq = QueryTest(
store,
'''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone''',
GN.nodeconstructor,
debug=debug)
if not tq.check([
None,
],
minrows=maxdrones + 1,
maxrows=maxdrones + 1,
delay=0.5,
maxtries=20):
sysenv.cma.cleanupwhendone = False
raise RuntimeError('Query of "up" status failed. Weirdness')
return sysenv, store
def __init__(self, _url):
if hasattr(self, 'session'):
raise SystemError('__init__ called too many times')
u = urlparse(_url)
if u.netloc.find('@') > -1:
credentials, self.host = u.netloc.split('@')
self.user, self.password, = credentials.split(':')
self.url = ''.join([u.scheme, '://', self.host, u.path, u.query])
neo4j.authenticate(self.host, self.user, self.password)
else:
self.url = _url
def connection():
if hasattr(connection, 'db'):
return connection.db
url = DATABASE_URL
u = urlparse(url)
if u.netloc.find('@') > -1:
credentials, host = u.netloc.split('@')
user, password, = credentials.split(':')
neo4j.authenticate(host, user, password)
url = ''.join([u.scheme, '://', host, u.path, u.query])
connection.db = neo4j.GraphDatabaseService(url)
return connection.db
def connection():
try:
return connection.db
except AttributeError:
url = DATABASE_URL
u = urlparse(url)
if u.netloc.find('@') > -1:
credentials, host = u.netloc.split('@')
user, password, = credentials.split(':')
neo4j.authenticate(host, user, password)
url = ''.join([u.scheme, '://', host, u.path, u.query])
connection.db = neo4j.GraphDatabaseService(url)
return connection.db
def connection():
if hasattr(connection, 'db'):
return connection.db
url = DATABASE_URL
u = urlparse(url)
if u.netloc.find('@') > -1:
credentials, host = u.netloc.split('@')
user, password, = credentials.split(':')
neo4j.authenticate(host, user, password)
url = ''.join([u.scheme, '://', host, u.path, u.query])
try:
connection.db = neo4j.GraphDatabaseService(url)
except SocketError as e:
raise SocketError("Error connecting to {0} - {1}".format(url, e))
return connection.db
def connection():
if hasattr(connection, 'db'):
return connection.db
url = DATABASE_URL
u = urlparse(url)
if u.netloc.find('@') > -1:
credentials, host = u.netloc.split('@')
user, password, = credentials.split(':')
neo4j.authenticate(host, user, password)
url = ''.join([u.scheme, '://', host, u.path, u.query])
try:
connection.db = neo4j.GraphDatabaseService(url)
except SocketError as e:
raise SocketError("Error connecting to {0} - {1}".format(url, e))
return connection.db
def connection():
if hasattr(connection, 'db'):
return connection.db
url = DATABASE_URL
u = urlparse(url)
if u.netloc.find('@') > -1:
credentials, host = u.netloc.split('@')
user, password, = credentials.split(':')
neo4j.authenticate(host, user, password)
url = ''.join([u.scheme, '://', host, u.path, u.query])
try:
connection.db = neo4j.GraphDatabaseService(url)
except SocketError as e:
raise SocketError("Error connecting to {0} - {1}".format(url, e))
if connection.db.neo4j_version >= (2, 0):
raise Exception("Support for neo4j 2.0 is in progress but not supported by this release.")
if connection.db.neo4j_version < (1, 8):
raise Exception("Versions of neo4j prior to 1.8 are unsupported.")
return connection.db
def __init__(self):
Util.__init__(self)
self.fdb = FastDataStore()
if os.environ.get('NEO4J_URL'):
graph_db_url = urlparse(os.environ.get('NEO4J_URL'))
neo4j.authenticate(
"{host}:{port}".format(host = graph_db_url.hostname, port = graph_db_url.port),
graph_db_url.username, graph_db_url.password
)
self.graphdb = neo4j.GraphDatabaseService(
'http://{host}:{port}/db/data'.format(host = graph_db_url.hostname, port = graph_db_url.port)
)
else:
self.graphdb = neo4j.GraphDatabaseService()
#self.graphdb.clear()
#print "cleared database!"
self.rel_key = "REL_CREATED"
self.node_key = "NODE_CREATED"
self.max_tries = 5
self.BATCH_LIM = 50
self.counter = 0
self.node_index = self.graphdb.get_or_create_index(neo4j.Node, 'NODE')
self.disambiguation_index = self.graphdb.get_or_create_index(neo4j.Node, self.DISAMBIGUATION)
def test_can_add_same_header_twice():
neo4j.authenticate("localhost:7474", "arthur", "excalibur")
neo4j.authenticate("localhost:7474", "arthur", "excalibur")
headers = neo4j._get_headers("localhost:7474")
assert headers['Authorization'] == 'Basic YXJ0aHVyOmV4Y2FsaWJ1cg=='
def test_can_add_same_header_twice():
neo4j.authenticate("localhost:7474", "arthur", "excalibur")
neo4j.authenticate("localhost:7474", "arthur", "excalibur")
headers = neo4j._get_headers("localhost:7474")
assert headers['Authorization'] == 'Basic YXJ0aHVyOmV4Y2FsaWJ1cg=='
from sklearn.datasets import load_iris
from sklearn.ensemble import RandomForestClassifier
import pandas as pd
import random
import json
from py2neo import neo4j
import numpy as np
from id3 import id3
from rf import rf
from collections import Counter
from ete2 import Tree
from random import randint
import math
neo4j.authenticate("http://*****:*****@localhost:7474/db/data/")
tipo = "personnage"
target = "nationalite"
vtarget = "Romain"
nnodes =400
cuenta1 = neo4j.CypherQuery(graph_db, "match (n:"+tipo+") where n."+target+" = '"+vtarget+ "' return count(n) as cuenta LIMIT "+str(nnodes)).execute()
cuenta2 = neo4j.CypherQuery(graph_db, "match (n:"+tipo+") where n."+target+" <> '"+vtarget+ "' return count(n) as cuenta LIMIT "+str(nnodes)).execute()
for c in cuenta1:
valor1 = c.cuenta
for c in cuenta2:
valor2 = c.cuenta
def test_authentication_adds_the_correct_header():
neo4j.authenticate("localhost:7474", "arthur", "excalibur")
headers = neo4j._get_headers("localhost:7474")
assert headers['Authorization'] == 'Basic YXJ0aHVyOmV4Y2FsaWJ1cg=='
def _graph_db(self):
host_port = "{0}:{1}".format(self._host, self._port)
uri = "{0}://{1}".format(self._scheme, host_port)
if self._user and self._password:
neo4j.authenticate(host_port, self._user, self._password)
return neo4j.ServiceRoot(uri).graph_db
def _graph(self):
host_port = "{0}:{1}".format(self._host, self._port)
uri = "{0}://{1}".format(self._scheme, host_port)
if self._user and self._password:
neo4j.authenticate(host_port, self._user, self._password)
return neo4j.ServiceRoot(uri).graph
def test_authentication_adds_the_correct_header():
neo4j.authenticate("localhost:7474", "arthur", "excalibur")
headers = neo4j._get_headers("localhost:7474")
assert headers['Authorization'] == 'Basic YXJ0aHVyOmV4Y2FsaWJ1cg=='
def __init__(self, bd, port, user, pss, label, ns, nd, l, m, traversals,
iteraciones):
self.nodes = []
self.ndim = nd
self.bd = bd
self.port = port
self.user = user
self.pss = pss
self.label = label
self.ns = ns
self.w_size = l
self.mode = m
self.iteraciones = iteraciones
# Setting up Neo4j DB
neo4j.authenticate("http://*****:*****@localhost:" +
str(self.port) +
"/db/data/")
batches = 100
if not os.path.exists("models/" + self.bd +
".npy") or not os.path.exists("models/" +
self.bd +
"l-degree.npy"):
print "Conecting to BD..."
nn = neo4j.CypherQuery(
self.graph_db, "match n return count(n) as cuenta1").execute()
self.numnodes = nn[0].cuenta1
self.sentences_array = []
nb = float(self.numnodes / batches)
count = -1
self.degree = []
for i in range(1, int(nb) + 1):
count += 1
consulta = "match (n)-[r]-(m) where n." + self.label + " <> '' return n,count(r) as d, n." + self.label + ", collect(m." + self.label + ") as collect skip " + str(
batches * count) + " limit " + str(batches)
cuenta = neo4j.CypherQuery(self.graph_db, consulta).execute()
print "\r" + str(float((i / nb) * 100)) + "%"
for cuenta1 in cuenta:
name = cuenta1['n.' + label].replace(" ", "_")
context = []
#Extracting context(relations)
for s in cuenta1['collect']:
if type(s) is list:
for x in s:
context.append(str(x).replace(" ", "_"))
else:
if s:
context.append(str(s).replace(" ", "_"))
#Extracting contexto(properties)
for t in cuenta1['n']:
s = cuenta1['n'][t]
if type(s) is list:
for x in s:
context.append(str(x).replace(" ", "_"))
else:
if s:
context.append(str(s).replace(" ", "_"))
if len(context) >= l - 1 and cuenta1.d is not None:
sentence = context
sentence.insert(0, name)
self.sentences_array.append(sentence)
self.degree.append(cuenta1.d)
np.save("models/" + self.bd, self.sentences_array)
np.save("models/" + self.bd + "l-degree", self.degree)
else:
self.sentences_array = np.load("models/" + self.bd + ".npy")
self.degree = np.load("models/" + self.bd + "l-degree.npy")
for s in self.sentences_array:
self.sentences[s[0]] = s[1:]
print "models/" + self.bd + ".npy"
import os
from flask import Flask
from py2neo import neo4j, cypher
from urlparse import urlparse
# this is how Heroku tells your app where to find the database.
if os.environ.get('NEO4J_URL'):
graph_db_url = urlparse(os.environ.get('NEO4J_URL'))
# set up authentication parameters
neo4j.authenticate("{host}:{port}".format(host=graph_db_url.hostname, port=graph_db_url.port), graph_db_url.username, graph_db_url.password)
# connect to authenticated graph database
graph_db = neo4j.GraphDatabaseService('http://{host}:{port}/db/data'.format(host=graph_db_url.hostname, port=graph_db_url.port))
else:
graph_db = neo4j.GraphDatabaseService('http://localhost:7474/db/data')
def create_graph(graph_db):
# Do we have a node that has a 'name' property, which is set to the value 'Neo'?
# We've probably been here before.
data, metadata = cypher.execute(graph_db, "START n=node(*) where n.name?='Neo' return n")
if data:
# Create two nodes, one for us and one for you.
# Make sure they both have 'name' properties with values.
from_node, to_node = graph_db.create({"name": "neo4j"}, {"name": "you"})
# create a 'loves' relationship from the 'from' node to the 'to' node
from_node.create_relationship_to(to_node, "loves")
print(node)
#TODO: apply the same to all 4 3asrs, after applying them in organizeForGraph.py; change rootdir to 'era'_raw_graph_data, and asr variable node to specific era. end result is a massive graph containing 4 3asrs, all their poets and whos born where, and who wrote what. and the actual poem sequence!
rootdir = '/home/ramez/python/capstone/app/andalsi_raw_graph_data'
#jsondir = '/home/ramez/python/capstone/app/jahili_json'
textfiles = glob.glob(os.path.join(rootdir, '*.txt'))
jsonfiles = glob.glob(os.path.join(rootdir, '*.json'))
authors = []
poem_list = []
poem_node_list = []
author_node_list = []
neo4j.authenticate("localhost:7474","neo4j","faisal")
#default for 7474 is empty. can add url.
graph_db = neo4j.Graph()
asr = Node("Era", era = "العصر الأندلسي")
graph_db.create(asr)
print(asr)
for f in textfiles:
title = linecache.getline(f,1)# filename, line_number
titles = title[1:-1].split(',')
author = linecache.getline(f,2)
# authors.append(author[:-1])
poet = Node("Poet", name = author)
def __init__(self,bd,port,user,pss,label,ns,nd,l,m,traversals,iteraciones):
self.nodes = []
self.ndim = nd
self.bd = bd
self.port = port
self.user = user
self.pss = pss
self.label = label
self.ns = ns
self.w_size = l
self.mode = m
self.iteraciones = iteraciones
# Setting up Neo4j DB
neo4j.authenticate("http://*****:*****@localhost:"+str(self.port)+"/db/data/")
batches = 100
if not os.path.exists("models/" + self.bd +".npy") or not os.path.exists("models/" + self.bd +"l-degree.npy"):
print "Conecting to BD..."
nn = neo4j.CypherQuery(self.graph_db, "match n return count(n) as cuenta1").execute()
self.numnodes = nn[0].cuenta1
self.sentences_array = []
nb = float(self.numnodes/batches)
count = -1
self.degree = []
for i in range(1,int(nb)+1):
count += 1
consulta = "match (n)-[r]-(m) where n."+self.label+" <> '' return n,count(r) as d, n."+self.label+", collect(m."+self.label+") as collect skip "+str(batches*count)+" limit "+str(batches)
cuenta = neo4j.CypherQuery(self.graph_db, consulta).execute()
print "\r"+str(float((i / nb)*100))+ "%"
for cuenta1 in cuenta:
name = cuenta1['n.'+label].replace(" ","_")
context = []
#Extracting context(relations)
for s in cuenta1['collect']:
if type(s) is list:
for x in s:
context.append(str(x).replace(" ","_"))
else:
if s:
context.append(str(s).replace(" ","_"))
#Extracting contexto(properties)
for t in cuenta1['n']:
s = cuenta1['n'][t]
if type(s) is list:
for x in s:
context.append(str(x).replace(" ","_"))
else:
if s:
context.append(str(s).replace(" ","_"))
if len(context) >= l-1 and cuenta1.d is not None:
sentence = context
sentence.insert(0,name)
self.sentences_array.append(sentence)
self.degree.append(cuenta1.d)
np.save("models/" + self.bd , self.sentences_array)
np.save("models/" + self.bd +"l-degree", self.degree)
else:
self.sentences_array = np.load("models/" + self.bd +".npy")
self.degree = np.load("models/" + self.bd +"l-degree.npy")
for s in self.sentences_array:
self.sentences[s[0]]=s[1:]
print "models/" + self.bd +".npy"
