def dbsetup(readonly=False): 'Set up our connection to Neo4j' ourstore = Store(neo4j.GraphDatabaseService(), uniqueindexmap={}, classkeymap={}) CMAinit(None, readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) return ourstore
def setup(dbhost='localhost', dbport=7474, dburl=None, querypath=None): ''' Program to set up for running our REST server. We do these things: - Attach to the database - Initialize our type objects so things like ClientQuery will work... - Load the queries into the database from flat files Not sure if doing this here makes the best sense, but it works, and currently we're the only one who cares about them so it seems reasonable -- at the moment ;-) Also we're most likely to iterate changing on those relating to the REST server, so fixing them just by restarting the REST server seems to make a lot of sense (at the moment) - Remember the set of queries in the 'allqueries' hash table ''' if dburl is None: dburl = ('http://%s:%d/db/data/' % (dbhost, dbport)) print >> sys.stderr, 'CREATING Graph("%s")' % dburl neodb = neo4j.Graph(dburl) qstore = Store(neodb, None, None) print GraphNode.classmap for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" if querypath is None: querypath = "/home/alanr/monitor/src/queries" queries = ClientQuery.load_tree(qstore, querypath) for q in queries: allqueries[q.queryname] = q qstore.commit() for q in allqueries: allqueries[q].bind_store(qstore)
def __init__(self, io, host='localhost', port=7474, cleanoutdb=False, debug=False , retries=300, readonly=False, encryption_required=False, use_network=True): 'Initialize and construct a global database instance' #print >> sys.stderr, 'CALLING NEW initglobal' CMAdb.log = logging.getLogger('cma') CMAdb.debug = debug CMAdb.io = io from hbring import HbRing syslog = logging.handlers.SysLogHandler(address='/dev/log' , facility=logging.handlers.SysLogHandler.LOG_DAEMON) syslog.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) CMAdb.log.addHandler(syslog) CMAdb.log.setLevel(logging.DEBUG) url = ('http://%s:%d/db/data/' % (host, port)) #print >> sys.stderr, 'CREATING GraphDatabaseService("%s")' % url neodb = neo4j.GraphDatabaseService(url) self.db = neodb if cleanoutdb: CMAdb.log.info('Re-initializing the NEO4j database') self.delete_all() self.db = neodb CMAdb.use_network = use_network trycount=0 while True: try: CMAdb.cdb = CMAdb(db=neodb) # Neo4j started. All is well with the world. break except (RuntimeError, IOError, py2neo.exceptions.ClientError) as exc: print >> sys.stderr, 'TRYING AGAIN [[%s]...[%s]' % (url, str(exc)) trycount += 1 if trycount > retries: print >> sys.stderr, ('Neo4j still not started - giving up.') CMAdb.log.critical('Neo4j still not started - giving up.') raise RuntimeError('Neo4j not running - giving up [%s]' % str(exc)) if (trycount % 60) == 1: print >> sys.stderr, ('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) CMAdb.log.warning('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) # Let's try again in a second... time.sleep(1) Store.debug = debug Store.log = CMAdb.log CMAdb.store = Store(neodb, CMAconsts.uniqueindexes, CMAconsts.classkeymap , readonly=readonly) if not readonly: for classname in GraphNode.classmap: GraphNode.initclasstypeobj(CMAdb.store, classname) from transaction import Transaction CMAdb.transaction = Transaction(encryption_required=encryption_required) #print >> sys.stderr, 'CMAdb:', CMAdb #print >> sys.stderr, 'CMAdb.store(cmadb.py):', CMAdb.store CMAdb.TheOneRing = CMAdb.store.load_or_create(HbRing, name='The_One_Ring' , ringtype=HbRing.THEONERING) CMAdb.transaction.commit_trans(io) #print >> sys.stderr, 'COMMITTING Store' #print >> sys.stderr, 'Transaction Commit results:', CMAdb.store.commit() CMAdb.store.commit() #print >> sys.stderr, 'Store COMMITTED' else: CMAdb.transaction = None
def __init__(self, io, host='localhost', port=7474, cleanoutdb=False, debug=False , retries=300, readonly=False, encryption_required=False, use_network=True): 'Initialize and construct a global database instance' #print >> sys.stderr, 'CALLING NEW initglobal' CMAdb.log = logging.getLogger('cma') CMAdb.debug = debug CMAdb.io = io from hbring import HbRing syslog = logging.handlers.SysLogHandler(address='/dev/log' , facility=logging.handlers.SysLogHandler.LOG_DAEMON) syslog.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) CMAdb.log.addHandler(syslog) CMAdb.log.setLevel(logging.DEBUG) url = ('http://%s:%d/db/data/' % (host, port)) #print >> sys.stderr, 'CREATING GraphDatabaseService("%s")' % url neodb = neo4j.GraphDatabaseService(url) self.db = neodb if cleanoutdb: CMAdb.log.info('Re-initializing the NEO4j database') self.delete_all() self.db = neodb CMAdb.use_network = use_network trycount=0 while True: try: CMAdb.cdb = CMAdb(db=neodb) # Neo4j started. All is well with the world. break except (RuntimeError, IOError, py2neo.exceptions.ClientError) as exc: print >> sys.stderr, 'TRYING AGAIN [[%s]...[%s]' % (url, str(exc)) trycount += 1 if trycount > retries: print >> sys.stderr, ('Neo4j still not started - giving up.') CMAdb.log.critical('Neo4j still not started - giving up.') raise RuntimeError('Neo4j not running - giving up [%s]' % str(exc)) if (trycount % 60) == 1: print >> sys.stderr, ('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) CMAdb.log.warning('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) # Let's try again in a second... time.sleep(1) Store.debug = debug Store.log = CMAdb.log CMAdb.store = Store(neodb, CMAconsts.uniqueindexes, CMAconsts.classkeymap , readonly=readonly) if not readonly: for classname in GraphNode.classmap: GraphNode.initclasstypeobj(CMAdb.store, classname) from transaction import Transaction CMAdb.transaction = Transaction(encryption_required=encryption_required) #print >> sys.stderr, 'CMAdb:', CMAdb #print >> sys.stderr, 'CMAdb.store(cmadb.py):', CMAdb.store CMAdb.TheOneRing = CMAdb.store.load_or_create(HbRing, name='The_One_Ring' , ringtype= HbRing.THEONERING) CMAdb.transaction.commit_trans(io) #print >> sys.stderr, 'COMMITTING Store' #print >> sys.stderr, 'Transaction Commit results:', CMAdb.store.commit() CMAdb.store.commit() #print >> sys.stderr, 'Store COMMITTED' else: CMAdb.transaction = None
def dbsetup(readonly=False, url=None): 'Set up our connection to Neo4j' if url is None: url = 'localhost.com:7474' Neo4jCreds().authenticate(url) ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={}) CMAinit(DummyIO(), readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) CMAdb.store = ourstore return ourstore
} } } ''' q3 = ClientQuery('ipowners', metadata3) Neo4jCreds().authenticate() neodb = neo4j.Graph() neodb.delete_all() umap = {'ClientQuery': True} ckmap = {'ClientQuery': {'index': 'ClientQuery', 'kattr':'queryname', 'value':'None'}} qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" queries = ClientQuery.load_tree(qstore, "%s/../queries" % (os.path.dirname(sys.argv[0]))) qlist = [q for q in queries] qstore.commit() print "%d node TREE LOADED!" % len(qlist) qe2 = qstore.load_or_create(ClientQuery, queryname='list') qe2.bind_store(qstore) testresult = '' for s in qe2.execute(None, idsonly=False, expandJSON=True): testresult += s print 'RESULT', testresult # Test out a command line query for s in qe2.cmdline_exec(None):
neodb = neo4j.GraphDatabaseService() neodb.clear() print >> sys.stderr, '========>classmap: %s' % (GraphNode.classmap) umap = {'ClientQuery': True} ckmap = { 'ClientQuery': { 'index': 'ClientQuery', 'kattr': 'queryname', 'value': 'None' } } qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" queries = ClientQuery.load_tree(qstore, "/home/alanr/monitor/src/queries") qlist = [q for q in queries] qstore.commit() print "%d node TREE LOADED!" % len(qlist) qe2 = qstore.load_or_create(ClientQuery, queryname='list') qe2.bind_store(qstore) testresult = '' for s in qe2.execute(None, idsonly=False, expandJSON=True): testresult += s print testresult # Test out a command line query for s in qe2.cmdline_exec(None): print s