def get_graph():
# Connect to graph
u = urlparse(session['neo4j_url'])
neo4j.authenticate('%s:%s' % (u.hostname, u.port), 'neo4j', session['neo4j_password'])
graph = neo4j.Graph("%s/db/data/" % session['neo4j_url'])
graph.cypher.execute('match (t:Tweet) return COUNT(t)')
return graph
def make_neo4jdb(species):
graph_db = neo4j.Graph()
# some bs about not authorized
# "Alternatively, authentication can be disabled completely by editing the value of
# the dbms.security.authorization_enabled
#setting in the conf/neo4j-server.properties file."
infile = os.getcwd() + '/names.csv'
qry = "LOAD CSV WITH HEADERS FROM 'file://%s' " % infile
qry += "AS line CREATE (g:gene {official_symbol: line.symbol} )"
ret = graph_db.cypher.execute(qry)
print "qry:", qry
print "ret:", ret
qry = "CREATE CONSTRAINT ON (g:gene) ASSERT g.official_symbol IS UNIQUE"
ret = graph_db.cypher.execute(qry)
print "qry:", qry
print "ret:", ret
infiles = glob.glob(os.getcwd() + '/pubmed_ids.*.csv')
for infile in infiles:
print infile
exp_label = infile.split("/")[-1].replace("pubmed_ids.",
"").replace(".csv", "")
qry = "USING PERIODIC COMMIT 500 "
qry += "LOAD CSV WITH HEADERS FROM 'file://%s' " % infile
qry += "AS line MATCH (g1:gene {official_symbol: line.symbol_1}), (g2:gene {official_symbol: line.symbol_2}) "
qry += "CREATE (g1)-[:%s {pubmed_ids: line.pubmed_ids}]->(g2) " % exp_label
ret = graph_db.cypher.execute(qry)
print "qry:", qry
print "ret:", ret
print "moving", neo4j_home + "/graph.db", "to", neo4j_home + "/" + species + "_graph.db"
shutil.move(neo4j_home + "/graph.db",
neo4j_home + "/" + species + "_graph.db")
def setup(dbhost='localhost', dbport=7474, dburl=None, querypath=None):
'''
Program to set up for running our REST server.
We do these things:
- Attach to the database
- Initialize our type objects so things like ClientQuery will work...
- Load the queries into the database from flat files
Not sure if doing this here makes the best sense, but it
works, and currently we're the only one who cares about them
so it seems reasonable -- at the moment ;-)
Also we're most likely to iterate changing on those relating to the
REST server, so fixing them just by restarting the REST server seems
to make a lot of sense (at the moment)
- Remember the set of queries in the 'allqueries' hash table
'''
if dburl is None:
dburl = ('http://%s:%d/db/data/' % (dbhost, dbport))
print >> sys.stderr, 'CREATING Graph("%s")' % dburl
neodb = neo4j.Graph(dburl)
qstore = Store(neodb, None, None)
print GraphNode.classmap
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(qstore, classname)
print "LOADING TREE!"
if querypath is None:
querypath = "/home/alanr/monitor/src/queries"
queries = ClientQuery.load_tree(qstore, querypath)
for q in queries:
allqueries[q.queryname] = q
qstore.commit()
for q in allqueries:
allqueries[q].bind_store(qstore)
def get_pop_ids(pop_url, query_params=list()):
"""
Retrieve list of PoPs uuids
:param pop_url: Url of the PoP DB
:param query_params: optional list of (key, value)s used as query parameters
:return list: List of PoPs uuids
"""
query = 'match node where node.type="{}" return node'.format('pop')
graph_db = neo4j.Graph(pop_url)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + pop_url)
results = []
for record in data.records:
prop = dict(record.node.properties)
filtered = False
for q in query_params:
if len(q[0]) > 0 and len(q[1]) > 0:
if q[0] == 'name' or q[0] == 'occi.epa.pop.name':
if prop['occi.epa.pop.name'].lower() != q[1].lower():
filtered = True
else:
tmp = repr(prop[q[0]]).lower()
if tmp != q[1].lower():
filtered = True
if not filtered:
results.append(prop['uuid'])
return results
def get_pop_links_target_uuid(pop_url, source_uuid):
"""
Return a list of tuple containing (target uuid, target type, link uuid)
for a given source uuid node in the PoP DB
:param pop_url: Url of the PoP DB
:param source_uuid: source uuid of the links to be retrieved
:return List: List of links information
"""
graph_db = neo4j.Graph(pop_url)
query = 'match n-[r]->m where n.type = "pop" and n.uuid="{}" ' \
' return r,m'.format(source_uuid)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + pop_url)
results = []
for record in data.records:
m_properties = dict(record.m.properties)
r_properties = dict(record.r.properties)
if 'uuid' in m_properties and 'uuid' in r_properties:
results.append((m_properties['uuid'], m_properties['type'].lower(),
r_properties['uuid']))
return results
def pytest_configure(config):
""" Session-wide test configuration.
Executes before any tests do and can be used to configure the test
environment.
To guard against accidental data loss ``pytest_configure`` will bail
instantly if the test db contains any data. It also configures it's own
logging which will override any previous user-configured logging. This
is to ensure the correct messags reach the users console during test run.
"""
db = neo4j.Graph(DEFAULT_DB)
rels = next(db.match(), None)
if rels:
logging.warning('Test Runner will only operate on an empty graph. '
'Either clear the DB with `neotool clear` or provide '
'a differnt DB URI ')
pytest.exit(1)
handler = ColorizingStreamHandler()
handler.level_map = PY2NEO_LOGGING_LEVEL_COLOUR_MAP
logger.addHandler(handler)
logger.info('- all logging captured at >= DEUBG and piped '
'to stdout for test failures. ')
logger.info('- tests running on Neo4J %s', db.neo4j_version)
def dbsetup(readonly=False):
'Set up our connection to Neo4j'
ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={})
CMAinit(DummyIO(), readonly=readonly, use_network=False)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(ourstore, classname)
return ourstore
def add_pop_edge(pop_url,
src_uuid,
trg_uuid,
timestamp,
label,
properties=None):
"""
Add new link between two PoPs
:param pop_url: Url of PoP DB
:param src_uuid: Source uuid
:param trg_uuid: Target uuid
:param timestamp: timestamp of the update
:param label: Label of the link
:param properties: optionally dict containing key values representing
PoP link Properties
"""
if not properties:
properties = dict()
graph_db = neo4j.Graph(pop_url)
src_index = ('pop', 'uuid', src_uuid)
trg_index = ('pop', 'uuid', trg_uuid)
db_src = neo_resource.get_node(graph_db, src_index)
db_trg = neo_resource.get_node(graph_db, trg_index)
properties['timestamp'] = timestamp
edge = Relationship.cast(db_src, label, db_trg, properties)
graph_db.create(edge)
def pytest_unconfigure(config):
""" Final tear-down behaviour of test environment
"""
db = neo4j.Graph(DEFAULT_DB)
db.clear()
logger.info('cleared db after tests completed')
def pytest_runtest_setup(item):
""" Pre-test configuration.
This will execute before each test (and after ``pytest_configure``) and
can be used to configure the environment on a test-by-test basis, or
even skip the test.
"""
# looking for markers like '@pytest.mark.neoversion("X.XX")'
version_markers = item.get_marker("neoversion")
if version_markers:
logger.info('minimum neo version required')
db = neo4j.Graph(DEFAULT_DB)
required_version = Decimal(version_markers.args[0])
version_tuple = db.neo4j_version[:-1] # ignore meta
actual_version = Decimal('{0}.{1}'.format(
version_tuple[0], ''.join(str(n) for n in version_tuple[1:])))
if required_version > actual_version:
logger.warning('{test_name} requires neo > {version}'.format(
test_name=item.name, version=required_version))
pytest.skip()
def get_resource(pop_url, pop_id, resource_type, uuid, physical=False):
"""
Retrieve properties of a given node and sanitize them given its type and
its unique ID
:param pop_url: Url of PoP DB
:param pop_id: PoP Name
:param resource_type: Type of the resource
:param uuid: Node uuid
:param physical: Boolean default False.
It is True if the node that should be retrieved is a physical node
:return dict: Node properties
"""
graph_url, pop = _get_graph_url(pop_url, pop_id)
graph_db = neo4j.Graph(graph_url)
if physical:
node_properties = _get_physical_resource_by_type_and_uuid(
graph_db, pop, resource_type, uuid)
else:
node_properties = _get_resource_by_type_and_uuid(
graph_db, pop, resource_type, uuid)
results = {}
for key in node_properties:
if isinstance(node_properties[key], str):
results[key] = node_properties[key].strip().lower()
else:
results[key] = node_properties[key]
return results
def get_graph():
global NEO4J_URL, NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD
# Connect to graph
neo4j.authenticate(NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD)
graph = neo4j.Graph(NEO4J_URL)
graph.cypher.execute('match (t:Tweet) return COUNT(t)')
return graph
def test_wrong_host_will_fail():
graph = neo4j.Graph("http://localtoast:7474/db/data/")
try:
graph.resource.get()
except NetworkAddressError:
assert True
else:
assert False
def test_wrong_path_will_fail():
graph = neo4j.Graph("http://localhost:7474/foo/bar/")
try:
graph.resource.get()
except neo4j.ClientError:
assert True
else:
assert False
def test_wrong_port_will_fail():
graph = neo4j.Graph("http://localhost:7575/db/data/")
try:
graph.resource.get()
except SocketError:
assert True
else:
assert False
def graph(request):
graph = neo4j.Graph(DEFAULT_DB)
try:
graph.clear()
except Exception as exc:
logger.exception('Failed to clear db')
return graph
def delete_pop(pop_url, uuid):
"""
Delete PoP with given uuid
:param pop_url: Url of PoP DB
:param uuid: PoP uuid
"""
graph_db = neo4j.Graph(pop_url)
index = ('pop', 'uuid', uuid)
neo_resource.delete_node(graph_db, index=index)
def get_clean_database():
# Constraints have to be removed before the indexed property keys can be removed.
graph = neo4j.Graph()
if graph.supports_node_labels:
for label in graph.node_labels:
for key in graph.schema.get_unique_constraints(label):
graph.schema.drop_unique_constraint(label, key)
for key in graph.schema.get_indexes(label):
graph.schema.drop_index(label, key)
return graph
def initenviron(logname,
maxdrones,
mgmtsystem,
debug=False,
cmaimage='',
nanoimages=[],
timeout=90,
nanodebug=0,
cmadebug=0):
'Initialize the test environment.'
logwatch = LogWatcher(logname, [],
timeout,
returnonlymatch=True,
debug=debug)
logwatch.setwatch()
sysenv = SystemTestEnvironment(logname,
maxdrones,
mgmtsystem,
cmaimage=cmaimage,
nanoimages=nanoimages,
nanodebug=nanodebug,
cmadebug=cmadebug)
CMAinit(None,
host=str(sysenv.cma.ipaddr),
readonly=True,
neologin=SystemTestEnvironment.NEO4JLOGIN,
neopass=SystemTestEnvironment.NEO4JPASS)
url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)
print >> sys.stderr, 'OPENING Neo4j at URL %s' % url
neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr,
SystemTestEnvironment.NEO4JLOGIN,
SystemTestEnvironment.NEO4JPASS)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
logger('$(grep MemFree: /proc/meminfo)', hardquote=False)
tq = QueryTest(
store,
'''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone''',
GN.nodeconstructor,
debug=debug)
if not tq.check([
None,
],
minrows=maxdrones + 1,
maxrows=maxdrones + 1,
delay=0.5,
maxtries=20):
sysenv.cma.cleanupwhendone = False
raise RuntimeError('Query of "up" status failed. Weirdness')
return sysenv, store
def dbsetup(readonly=False, url=None):
'Set up our connection to Neo4j'
if url is None:
url = 'localhost.com:7474'
Neo4jCreds().authenticate(url)
ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={})
CMAinit(DummyIO(), readonly=readonly, use_network=False)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(ourstore, classname)
CMAdb.store = ourstore
return ourstore
def get_resource_openstack_ids(pop_url,
pop_id,
resource_type,
query_params=list()):
"""
Retrive list of nodes uuid of a given type
:param pop_url: Url of PoP DB
:param pop_id: PoP ID
:param resource_type: type of nodes that should be retrieved
:param query_params: optional list of (key, value)s used as query parameters
:return list: list of nodes uuids
"""
graph_url, pop = _get_graph_url(pop_url, pop_id)
query = 'match node where node.type="{}" and node.pop="{}" '.format(
resource_type, pop)
for q in query_params:
if len(q[0]) > 0 and len(q[1]) > 0:
val = '(?i).*%s.*' % (q[1])
query += 'and node.attributes=~"{}"'.format(val)
query += ' return node.openstack_uuid'
graph_db = neo4j.Graph(graph_url)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + graph_url)
results = []
for record in data.records:
if record['node.openstack_uuid']:
results.append(record['node.openstack_uuid'])
query = 'match node where node.type=~"(?i){}" and node.pop="{}" '.format(
resource_type, pop)
for q in query_params:
if len(q[0]) > 0 and len(q[1]) > 0:
val = '(?i).*%s.*' % (q[1])
query += 'and node.attributes=~"{}"'.format(val)
query += ' return node.physical_name'
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + graph_url)
for record in data.records:
if record['node.physical_name']:
results.append(record['node.physical_name'])
return results
def get_pop(pop_url, uuid):
"""
Retrieve PoP given the uuid
:param pop_url: Url of PoP DB
:param uuid: PoP uuid
:return dict: Dictionary containing PoP properties
"""
graph_db = neo4j.Graph(pop_url)
index = ('pop', 'uuid', uuid)
pop = neo_resource.get_node(graph_db, index)
if pop:
results = dict(pop.properties)
return results
raise HTTPError(404, 'Resource not found: /pop/' + uuid)
def testmain(logname, maxdrones=25, debug=False):
'A simple test main program'
regexes = []
#pylint says: [W0612:testmain] Unused variable 'j'
#pylint: disable=W0612
for j in range(0,maxdrones+1):
regexes.append('Stored packages JSON data from *([^ ]*) ')
logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True)
logwatch.setwatch()
sysenv = SystemTestEnvironment(maxdrones)
print >> sys.stderr, 'Systems all up and running.'
url = ('http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474))
CMAinit(None)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
results = logwatch.lookforall()
if debug:
print >> sys.stderr, 'WATCH RESULTS:', results
tq = QueryTest(store
, "START drone=node:Drone('*:*') RETURN drone"
, GN.nodeconstructor, debug=debug)
print >> sys.stderr, 'Running Query'
if tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1):
print 'WOOT! Systems passed query check after initial startup!'
else:
print 'Systems FAILED initial startup query check'
print 'Do you have a second CMA running??'
print 'Rerunning query with debug=True'
tq.debug = True
tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1)
return 1
cma = sysenv.cma
nano = sysenv.nanoprobes[0]
regex = (r'%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown'
% (cma.hostname, nano.hostname, nano.ipaddr))
#print >> sys.stderr, 'REGEX IS: [%s]' % regex
logwatch = LogWatcher(logname, [regex,], timeout=30, returnonlymatch=False)
logwatch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
logwatch.look()
time.sleep(30)
tq = QueryTest(store
, ('''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" RETURN drone''')
, GN.nodeconstructor, debug=debug)
if tq.check([nano,], downbyshutdown, maxrows=1):
print 'WOOT! Systems passed query check after nano shutdown!'
else:
print 'Systems FAILED query check after nano shutdown'
def initstore():
global version_printed
db = neo4j.Graph(None)
if not version_printed:
print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version)
print >> sys.stderr, 'USING py2neo VERSION %s' % str(
py2neo.__version__)
version_printed = True
GraphNode.clean_graphnodes()
db.delete_all()
CMAinit(None)
OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
OurStore.clean_store()
CreateIndexes(db, [cls.__name__ for cls in Classes])
return OurStore
def update_pop(pop_url, uuid, timestamp, properties=None):
"""
Update a node in PoP DB representing a Point Of Presence
:param pop_url: Url of PoP DB
:param uuid: Uuid of the node to be updated
:param timestamp: timestamp of the update
:param properties: optionally dict containing key values representing new
PoP Properties
"""
if not properties:
properties = dict()
properties['type'] = 'pop'
index = ('pop', 'uuid', uuid)
graph_db = neo4j.Graph(pop_url)
neo_resource.update_node(graph_db, index, timestamp, properties=properties)
def test_is_not_idempotent():
graph = neo4j.Graph()
alice, = graph.create({"name": "Alice"})
batch = neo4j.WriteBatch(graph)
batch.create_path(alice, "KNOWS", {"name": "Bob"})
results = batch.submit()
path = results[0]
bob = path.nodes[1]
assert path.nodes[0] == alice
assert bob["name"] == "Bob"
batch = neo4j.WriteBatch(graph)
batch.create_path(alice, "KNOWS", {"name": "Bob"})
results = batch.submit()
path = results[0]
assert path.nodes[0] == alice
assert path.nodes[1] != bob
def get_links_target_uuid(pop_url, pop_id, source_uuid):
"""
Retrieve Link target uuids given link's source uuid
:param pop_url: Url of PoP DB
:param pop_id: PoP ID
:param source_uuid: link's source uuid
:return list: List of link's targets UUIDs
"""
graph_url, pop = _get_graph_url(pop_url, pop_id)
graph_db = neo4j.Graph(graph_url)
query = 'match n-[r]->m where n.openstack_uuid="{}" ' \
' return m'.format(source_uuid)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + graph_url)
results = []
for record in data.records:
m_properties = dict(record.m.properties)
if 'openstack_uuid' in m_properties:
results.append(
(m_properties['openstack_uuid'], m_properties['type'].lower()))
if 'physical_name' in m_properties:
results.append(
(m_properties['physical_name'], m_properties['type'].lower()))
query = 'match n-[r]->m where n.physical_name="{}" ' \
' return m'.format(source_uuid)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + graph_url)
for record in data.records:
m_properties = dict(record.m.properties)
if 'openstack_uuid' in m_properties:
results.append(
(m_properties['openstack_uuid'], m_properties['type'].lower()))
if 'physical_name' in m_properties:
results.append(
(m_properties['physical_name'], m_properties['type'].lower()))
return results
def get_pop_link_source_target(pop_url, uuid):
"""
Return the source and the target uuid for a link
specified with it uuid
:param pop_url: Url of the PoP DB
:param uuid: Link uuid
:return tuple: (source uuid, target uuid)
"""
graph_db = neo4j.Graph(pop_url)
query = 'match n-[r]-m where r.uuid = "{}" return n.uuid, m.uuid'.format(
uuid)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + pop_url)
for record in data.records:
return record['n.uuid'], record['m.uuid']
def get_pop_link(pop_url, uuid):
"""
Return link properites and link label
:param pop_url: Url of the PoP DB
:param uuid: Link uuid
:return tuple: (link properties, link label)
"""
graph_db = neo4j.Graph(pop_url)
query = 'match n-[r]-m where r.uuid = "{}" return r'.format(uuid)
try:
data = graph_db.cypher.execute(query)
except Exception:
raise HTTPError(400, "Error connecting to graph url " + pop_url)
for record in data.records:
r_properties = dict(record.r.properties)
return r_properties, record.r.type
def on_status(self, status):
# print status.user.location
if self.first_run:
self.first_run = False
self.graph = neo4j.Graph(self.url)
if status.user.location:
if any(x in status.user.location for x in self.md):
# print status._json
self.results = self.results + 1
print "Current Results: " + str(self.results)
self.twts.append(status._json)
if len(self.twts) == 10:
print "Importing tweets..."
with open("dataset.json", "a") as myfile:
json.dump(self.twts, myfile)
self.graph.cypher.execute(self.query, tweets=self.twts)
self.twts = []
