def get_graph(): # Connect to graph u = urlparse(session['neo4j_url']) neo4j.authenticate('%s:%s' % (u.hostname, u.port), 'neo4j', session['neo4j_password']) graph = neo4j.Graph("%s/db/data/" % session['neo4j_url']) graph.cypher.execute('match (t:Tweet) return COUNT(t)') return graph
def make_neo4jdb(species): graph_db = neo4j.Graph() # some bs about not authorized # "Alternatively, authentication can be disabled completely by editing the value of # the dbms.security.authorization_enabled #setting in the conf/neo4j-server.properties file." infile = os.getcwd() + '/names.csv' qry = "LOAD CSV WITH HEADERS FROM 'file://%s' " % infile qry += "AS line CREATE (g:gene {official_symbol: line.symbol} )" ret = graph_db.cypher.execute(qry) print "qry:", qry print "ret:", ret qry = "CREATE CONSTRAINT ON (g:gene) ASSERT g.official_symbol IS UNIQUE" ret = graph_db.cypher.execute(qry) print "qry:", qry print "ret:", ret infiles = glob.glob(os.getcwd() + '/pubmed_ids.*.csv') for infile in infiles: print infile exp_label = infile.split("/")[-1].replace("pubmed_ids.", "").replace(".csv", "") qry = "USING PERIODIC COMMIT 500 " qry += "LOAD CSV WITH HEADERS FROM 'file://%s' " % infile qry += "AS line MATCH (g1:gene {official_symbol: line.symbol_1}), (g2:gene {official_symbol: line.symbol_2}) " qry += "CREATE (g1)-[:%s {pubmed_ids: line.pubmed_ids}]->(g2) " % exp_label ret = graph_db.cypher.execute(qry) print "qry:", qry print "ret:", ret print "moving", neo4j_home + "/graph.db", "to", neo4j_home + "/" + species + "_graph.db" shutil.move(neo4j_home + "/graph.db", neo4j_home + "/" + species + "_graph.db")
def setup(dbhost='localhost', dbport=7474, dburl=None, querypath=None): ''' Program to set up for running our REST server. We do these things: - Attach to the database - Initialize our type objects so things like ClientQuery will work... - Load the queries into the database from flat files Not sure if doing this here makes the best sense, but it works, and currently we're the only one who cares about them so it seems reasonable -- at the moment ;-) Also we're most likely to iterate changing on those relating to the REST server, so fixing them just by restarting the REST server seems to make a lot of sense (at the moment) - Remember the set of queries in the 'allqueries' hash table ''' if dburl is None: dburl = ('http://%s:%d/db/data/' % (dbhost, dbport)) print >> sys.stderr, 'CREATING Graph("%s")' % dburl neodb = neo4j.Graph(dburl) qstore = Store(neodb, None, None) print GraphNode.classmap for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" if querypath is None: querypath = "/home/alanr/monitor/src/queries" queries = ClientQuery.load_tree(qstore, querypath) for q in queries: allqueries[q.queryname] = q qstore.commit() for q in allqueries: allqueries[q].bind_store(qstore)
def get_pop_ids(pop_url, query_params=list()): """ Retrieve list of PoPs uuids :param pop_url: Url of the PoP DB :param query_params: optional list of (key, value)s used as query parameters :return list: List of PoPs uuids """ query = 'match node where node.type="{}" return node'.format('pop') graph_db = neo4j.Graph(pop_url) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + pop_url) results = [] for record in data.records: prop = dict(record.node.properties) filtered = False for q in query_params: if len(q[0]) > 0 and len(q[1]) > 0: if q[0] == 'name' or q[0] == 'occi.epa.pop.name': if prop['occi.epa.pop.name'].lower() != q[1].lower(): filtered = True else: tmp = repr(prop[q[0]]).lower() if tmp != q[1].lower(): filtered = True if not filtered: results.append(prop['uuid']) return results
def get_pop_links_target_uuid(pop_url, source_uuid): """ Return a list of tuple containing (target uuid, target type, link uuid) for a given source uuid node in the PoP DB :param pop_url: Url of the PoP DB :param source_uuid: source uuid of the links to be retrieved :return List: List of links information """ graph_db = neo4j.Graph(pop_url) query = 'match n-[r]->m where n.type = "pop" and n.uuid="{}" ' \ ' return r,m'.format(source_uuid) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + pop_url) results = [] for record in data.records: m_properties = dict(record.m.properties) r_properties = dict(record.r.properties) if 'uuid' in m_properties and 'uuid' in r_properties: results.append((m_properties['uuid'], m_properties['type'].lower(), r_properties['uuid'])) return results
def pytest_configure(config): """ Session-wide test configuration. Executes before any tests do and can be used to configure the test environment. To guard against accidental data loss ``pytest_configure`` will bail instantly if the test db contains any data. It also configures it's own logging which will override any previous user-configured logging. This is to ensure the correct messags reach the users console during test run. """ db = neo4j.Graph(DEFAULT_DB) rels = next(db.match(), None) if rels: logging.warning('Test Runner will only operate on an empty graph. ' 'Either clear the DB with `neotool clear` or provide ' 'a differnt DB URI ') pytest.exit(1) handler = ColorizingStreamHandler() handler.level_map = PY2NEO_LOGGING_LEVEL_COLOUR_MAP logger.addHandler(handler) logger.info('- all logging captured at >= DEUBG and piped ' 'to stdout for test failures. ') logger.info('- tests running on Neo4J %s', db.neo4j_version)
def dbsetup(readonly=False): 'Set up our connection to Neo4j' ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={}) CMAinit(DummyIO(), readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) return ourstore
def add_pop_edge(pop_url, src_uuid, trg_uuid, timestamp, label, properties=None): """ Add new link between two PoPs :param pop_url: Url of PoP DB :param src_uuid: Source uuid :param trg_uuid: Target uuid :param timestamp: timestamp of the update :param label: Label of the link :param properties: optionally dict containing key values representing PoP link Properties """ if not properties: properties = dict() graph_db = neo4j.Graph(pop_url) src_index = ('pop', 'uuid', src_uuid) trg_index = ('pop', 'uuid', trg_uuid) db_src = neo_resource.get_node(graph_db, src_index) db_trg = neo_resource.get_node(graph_db, trg_index) properties['timestamp'] = timestamp edge = Relationship.cast(db_src, label, db_trg, properties) graph_db.create(edge)
def pytest_unconfigure(config): """ Final tear-down behaviour of test environment """ db = neo4j.Graph(DEFAULT_DB) db.clear() logger.info('cleared db after tests completed')
def pytest_runtest_setup(item): """ Pre-test configuration. This will execute before each test (and after ``pytest_configure``) and can be used to configure the environment on a test-by-test basis, or even skip the test. """ # looking for markers like '@pytest.mark.neoversion("X.XX")' version_markers = item.get_marker("neoversion") if version_markers: logger.info('minimum neo version required') db = neo4j.Graph(DEFAULT_DB) required_version = Decimal(version_markers.args[0]) version_tuple = db.neo4j_version[:-1] # ignore meta actual_version = Decimal('{0}.{1}'.format( version_tuple[0], ''.join(str(n) for n in version_tuple[1:]))) if required_version > actual_version: logger.warning('{test_name} requires neo > {version}'.format( test_name=item.name, version=required_version)) pytest.skip()
def get_resource(pop_url, pop_id, resource_type, uuid, physical=False): """ Retrieve properties of a given node and sanitize them given its type and its unique ID :param pop_url: Url of PoP DB :param pop_id: PoP Name :param resource_type: Type of the resource :param uuid: Node uuid :param physical: Boolean default False. It is True if the node that should be retrieved is a physical node :return dict: Node properties """ graph_url, pop = _get_graph_url(pop_url, pop_id) graph_db = neo4j.Graph(graph_url) if physical: node_properties = _get_physical_resource_by_type_and_uuid( graph_db, pop, resource_type, uuid) else: node_properties = _get_resource_by_type_and_uuid( graph_db, pop, resource_type, uuid) results = {} for key in node_properties: if isinstance(node_properties[key], str): results[key] = node_properties[key].strip().lower() else: results[key] = node_properties[key] return results
def get_graph(): global NEO4J_URL, NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD # Connect to graph neo4j.authenticate(NEO4J_HOST_PORT, NEO4J_USER, NEO4J_PASSWORD) graph = neo4j.Graph(NEO4J_URL) graph.cypher.execute('match (t:Tweet) return COUNT(t)') return graph
def test_wrong_host_will_fail(): graph = neo4j.Graph("http://localtoast:7474/db/data/") try: graph.resource.get() except NetworkAddressError: assert True else: assert False
def test_wrong_path_will_fail(): graph = neo4j.Graph("http://localhost:7474/foo/bar/") try: graph.resource.get() except neo4j.ClientError: assert True else: assert False
def test_wrong_port_will_fail(): graph = neo4j.Graph("http://localhost:7575/db/data/") try: graph.resource.get() except SocketError: assert True else: assert False
def graph(request): graph = neo4j.Graph(DEFAULT_DB) try: graph.clear() except Exception as exc: logger.exception('Failed to clear db') return graph
def delete_pop(pop_url, uuid): """ Delete PoP with given uuid :param pop_url: Url of PoP DB :param uuid: PoP uuid """ graph_db = neo4j.Graph(pop_url) index = ('pop', 'uuid', uuid) neo_resource.delete_node(graph_db, index=index)
def get_clean_database(): # Constraints have to be removed before the indexed property keys can be removed. graph = neo4j.Graph() if graph.supports_node_labels: for label in graph.node_labels: for key in graph.schema.get_unique_constraints(label): graph.schema.drop_unique_constraint(label, key) for key in graph.schema.get_indexes(label): graph.schema.drop_index(label, key) return graph
def initenviron(logname, maxdrones, mgmtsystem, debug=False, cmaimage='', nanoimages=[], timeout=90, nanodebug=0, cmadebug=0): 'Initialize the test environment.' logwatch = LogWatcher(logname, [], timeout, returnonlymatch=True, debug=debug) logwatch.setwatch() sysenv = SystemTestEnvironment(logname, maxdrones, mgmtsystem, cmaimage=cmaimage, nanoimages=nanoimages, nanodebug=nanodebug, cmadebug=cmadebug) CMAinit(None, host=str(sysenv.cma.ipaddr), readonly=True, neologin=SystemTestEnvironment.NEO4JLOGIN, neopass=SystemTestEnvironment.NEO4JPASS) url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474) print >> sys.stderr, 'OPENING Neo4j at URL %s' % url neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr, SystemTestEnvironment.NEO4JLOGIN, SystemTestEnvironment.NEO4JPASS) store = Store(neo4j.Graph(url), readonly=True) for classname in GN.GraphNode.classmap: GN.GraphNode.initclasstypeobj(store, classname) logger('$(grep MemFree: /proc/meminfo)', hardquote=False) tq = QueryTest( store, '''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone''', GN.nodeconstructor, debug=debug) if not tq.check([ None, ], minrows=maxdrones + 1, maxrows=maxdrones + 1, delay=0.5, maxtries=20): sysenv.cma.cleanupwhendone = False raise RuntimeError('Query of "up" status failed. Weirdness') return sysenv, store
def dbsetup(readonly=False, url=None): 'Set up our connection to Neo4j' if url is None: url = 'localhost.com:7474' Neo4jCreds().authenticate(url) ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={}) CMAinit(DummyIO(), readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) CMAdb.store = ourstore return ourstore
def get_resource_openstack_ids(pop_url, pop_id, resource_type, query_params=list()): """ Retrive list of nodes uuid of a given type :param pop_url: Url of PoP DB :param pop_id: PoP ID :param resource_type: type of nodes that should be retrieved :param query_params: optional list of (key, value)s used as query parameters :return list: list of nodes uuids """ graph_url, pop = _get_graph_url(pop_url, pop_id) query = 'match node where node.type="{}" and node.pop="{}" '.format( resource_type, pop) for q in query_params: if len(q[0]) > 0 and len(q[1]) > 0: val = '(?i).*%s.*' % (q[1]) query += 'and node.attributes=~"{}"'.format(val) query += ' return node.openstack_uuid' graph_db = neo4j.Graph(graph_url) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + graph_url) results = [] for record in data.records: if record['node.openstack_uuid']: results.append(record['node.openstack_uuid']) query = 'match node where node.type=~"(?i){}" and node.pop="{}" '.format( resource_type, pop) for q in query_params: if len(q[0]) > 0 and len(q[1]) > 0: val = '(?i).*%s.*' % (q[1]) query += 'and node.attributes=~"{}"'.format(val) query += ' return node.physical_name' try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + graph_url) for record in data.records: if record['node.physical_name']: results.append(record['node.physical_name']) return results
def get_pop(pop_url, uuid): """ Retrieve PoP given the uuid :param pop_url: Url of PoP DB :param uuid: PoP uuid :return dict: Dictionary containing PoP properties """ graph_db = neo4j.Graph(pop_url) index = ('pop', 'uuid', uuid) pop = neo_resource.get_node(graph_db, index) if pop: results = dict(pop.properties) return results raise HTTPError(404, 'Resource not found: /pop/' + uuid)
def testmain(logname, maxdrones=25, debug=False): 'A simple test main program' regexes = [] #pylint says: [W0612:testmain] Unused variable 'j' #pylint: disable=W0612 for j in range(0,maxdrones+1): regexes.append('Stored packages JSON data from *([^ ]*) ') logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True) logwatch.setwatch() sysenv = SystemTestEnvironment(maxdrones) print >> sys.stderr, 'Systems all up and running.' url = ('http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)) CMAinit(None) store = Store(neo4j.Graph(url), readonly=True) for classname in GN.GraphNode.classmap: GN.GraphNode.initclasstypeobj(store, classname) results = logwatch.lookforall() if debug: print >> sys.stderr, 'WATCH RESULTS:', results tq = QueryTest(store , "START drone=node:Drone('*:*') RETURN drone" , GN.nodeconstructor, debug=debug) print >> sys.stderr, 'Running Query' if tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1): print 'WOOT! Systems passed query check after initial startup!' else: print 'Systems FAILED initial startup query check' print 'Do you have a second CMA running??' print 'Rerunning query with debug=True' tq.debug = True tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1) return 1 cma = sysenv.cma nano = sysenv.nanoprobes[0] regex = (r'%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown' % (cma.hostname, nano.hostname, nano.ipaddr)) #print >> sys.stderr, 'REGEX IS: [%s]' % regex logwatch = LogWatcher(logname, [regex,], timeout=30, returnonlymatch=False) logwatch.setwatch() nano.stopservice(SystemTestEnvironment.NANOSERVICE) logwatch.look() time.sleep(30) tq = QueryTest(store , ('''START drone=node:Drone('*:*') ''' '''WHERE drone.designation = "{0.hostname}" RETURN drone''') , GN.nodeconstructor, debug=debug) if tq.check([nano,], downbyshutdown, maxrows=1): print 'WOOT! Systems passed query check after nano shutdown!' else: print 'Systems FAILED query check after nano shutdown'
def initstore(): global version_printed db = neo4j.Graph(None) if not version_printed: print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version) print >> sys.stderr, 'USING py2neo VERSION %s' % str( py2neo.__version__) version_printed = True GraphNode.clean_graphnodes() db.delete_all() CMAinit(None) OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap) OurStore.clean_store() CreateIndexes(db, [cls.__name__ for cls in Classes]) return OurStore
def update_pop(pop_url, uuid, timestamp, properties=None): """ Update a node in PoP DB representing a Point Of Presence :param pop_url: Url of PoP DB :param uuid: Uuid of the node to be updated :param timestamp: timestamp of the update :param properties: optionally dict containing key values representing new PoP Properties """ if not properties: properties = dict() properties['type'] = 'pop' index = ('pop', 'uuid', uuid) graph_db = neo4j.Graph(pop_url) neo_resource.update_node(graph_db, index, timestamp, properties=properties)
def test_is_not_idempotent(): graph = neo4j.Graph() alice, = graph.create({"name": "Alice"}) batch = neo4j.WriteBatch(graph) batch.create_path(alice, "KNOWS", {"name": "Bob"}) results = batch.submit() path = results[0] bob = path.nodes[1] assert path.nodes[0] == alice assert bob["name"] == "Bob" batch = neo4j.WriteBatch(graph) batch.create_path(alice, "KNOWS", {"name": "Bob"}) results = batch.submit() path = results[0] assert path.nodes[0] == alice assert path.nodes[1] != bob
def get_links_target_uuid(pop_url, pop_id, source_uuid): """ Retrieve Link target uuids given link's source uuid :param pop_url: Url of PoP DB :param pop_id: PoP ID :param source_uuid: link's source uuid :return list: List of link's targets UUIDs """ graph_url, pop = _get_graph_url(pop_url, pop_id) graph_db = neo4j.Graph(graph_url) query = 'match n-[r]->m where n.openstack_uuid="{}" ' \ ' return m'.format(source_uuid) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + graph_url) results = [] for record in data.records: m_properties = dict(record.m.properties) if 'openstack_uuid' in m_properties: results.append( (m_properties['openstack_uuid'], m_properties['type'].lower())) if 'physical_name' in m_properties: results.append( (m_properties['physical_name'], m_properties['type'].lower())) query = 'match n-[r]->m where n.physical_name="{}" ' \ ' return m'.format(source_uuid) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + graph_url) for record in data.records: m_properties = dict(record.m.properties) if 'openstack_uuid' in m_properties: results.append( (m_properties['openstack_uuid'], m_properties['type'].lower())) if 'physical_name' in m_properties: results.append( (m_properties['physical_name'], m_properties['type'].lower())) return results
def get_pop_link_source_target(pop_url, uuid): """ Return the source and the target uuid for a link specified with it uuid :param pop_url: Url of the PoP DB :param uuid: Link uuid :return tuple: (source uuid, target uuid) """ graph_db = neo4j.Graph(pop_url) query = 'match n-[r]-m where r.uuid = "{}" return n.uuid, m.uuid'.format( uuid) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + pop_url) for record in data.records: return record['n.uuid'], record['m.uuid']
def get_pop_link(pop_url, uuid): """ Return link properites and link label :param pop_url: Url of the PoP DB :param uuid: Link uuid :return tuple: (link properties, link label) """ graph_db = neo4j.Graph(pop_url) query = 'match n-[r]-m where r.uuid = "{}" return r'.format(uuid) try: data = graph_db.cypher.execute(query) except Exception: raise HTTPError(400, "Error connecting to graph url " + pop_url) for record in data.records: r_properties = dict(record.r.properties) return r_properties, record.r.type
def on_status(self, status): # print status.user.location if self.first_run: self.first_run = False self.graph = neo4j.Graph(self.url) if status.user.location: if any(x in status.user.location for x in self.md): # print status._json self.results = self.results + 1 print "Current Results: " + str(self.results) self.twts.append(status._json) if len(self.twts) == 10: print "Importing tweets..." with open("dataset.json", "a") as myfile: json.dump(self.twts, myfile) self.graph.cypher.execute(self.query, tweets=self.twts) self.twts = []