def __init__(self, ipaddr): GraphNode.__init__(self, domain='global') if isinstance(ipaddr, str): ipaddr = pyNetAddr(ipaddr) if isinstance(ipaddr, pyNetAddr): ipaddr = ipaddr.toIPv6() self.ipaddr = str(ipaddr)
def dbsetup(readonly=False): 'Set up our connection to Neo4j' ourstore = Store(neo4j.GraphDatabaseService(), uniqueindexmap={}, classkeymap={}) CMAinit(None, readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) return ourstore
def setup(dbhost='localhost', dbport=7474, dburl=None, querypath=None): ''' Program to set up for running our REST server. We do these things: - Attach to the database - Initialize our type objects so things like ClientQuery will work... - Load the queries into the database from flat files Not sure if doing this here makes the best sense, but it works, and currently we're the only one who cares about them so it seems reasonable -- at the moment ;-) Also we're most likely to iterate changing on those relating to the REST server, so fixing them just by restarting the REST server seems to make a lot of sense (at the moment) - Remember the set of queries in the 'allqueries' hash table ''' if dburl is None: dburl = ('http://%s:%d/db/data/' % (dbhost, dbport)) print >> sys.stderr, 'CREATING Graph("%s")' % dburl neodb = neo4j.Graph(dburl) qstore = Store(neodb, None, None) print GraphNode.classmap for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" if querypath is None: querypath = "/home/alanr/monitor/src/queries" queries = ClientQuery.load_tree(qstore, querypath) for q in queries: allqueries[q.queryname] = q qstore.commit() for q in allqueries: allqueries[q].bind_store(qstore)
def __init__(self, io, host='localhost', port=7474, cleanoutdb=False, debug=False , retries=300, readonly=False, encryption_required=False, use_network=True): 'Initialize and construct a global database instance' #print >> sys.stderr, 'CALLING NEW initglobal' CMAdb.log = logging.getLogger('cma') CMAdb.debug = debug CMAdb.io = io from hbring import HbRing syslog = logging.handlers.SysLogHandler(address='/dev/log' , facility=logging.handlers.SysLogHandler.LOG_DAEMON) syslog.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) CMAdb.log.addHandler(syslog) CMAdb.log.setLevel(logging.DEBUG) url = ('http://%s:%d/db/data/' % (host, port)) #print >> sys.stderr, 'CREATING GraphDatabaseService("%s")' % url neodb = neo4j.GraphDatabaseService(url) self.db = neodb if cleanoutdb: CMAdb.log.info('Re-initializing the NEO4j database') self.delete_all() self.db = neodb CMAdb.use_network = use_network trycount=0 while True: try: CMAdb.cdb = CMAdb(db=neodb) # Neo4j started. All is well with the world. break except (RuntimeError, IOError, py2neo.exceptions.ClientError) as exc: print >> sys.stderr, 'TRYING AGAIN [[%s]...[%s]' % (url, str(exc)) trycount += 1 if trycount > retries: print >> sys.stderr, ('Neo4j still not started - giving up.') CMAdb.log.critical('Neo4j still not started - giving up.') raise RuntimeError('Neo4j not running - giving up [%s]' % str(exc)) if (trycount % 60) == 1: print >> sys.stderr, ('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) CMAdb.log.warning('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) # Let's try again in a second... time.sleep(1) Store.debug = debug Store.log = CMAdb.log CMAdb.store = Store(neodb, CMAconsts.uniqueindexes, CMAconsts.classkeymap , readonly=readonly) if not readonly: for classname in GraphNode.classmap: GraphNode.initclasstypeobj(CMAdb.store, classname) from transaction import Transaction CMAdb.transaction = Transaction(encryption_required=encryption_required) #print >> sys.stderr, 'CMAdb:', CMAdb #print >> sys.stderr, 'CMAdb.store(cmadb.py):', CMAdb.store CMAdb.TheOneRing = CMAdb.store.load_or_create(HbRing, name='The_One_Ring' , ringtype= HbRing.THEONERING) CMAdb.transaction.commit_trans(io) #print >> sys.stderr, 'COMMITTING Store' #print >> sys.stderr, 'Transaction Commit results:', CMAdb.store.commit() CMAdb.store.commit() #print >> sys.stderr, 'Store COMMITTED' else: CMAdb.transaction = None
def __init__(self, io, host='localhost', port=7474, cleanoutdb=False, debug=False , retries=300, readonly=False, encryption_required=False, use_network=True): 'Initialize and construct a global database instance' #print >> sys.stderr, 'CALLING NEW initglobal' CMAdb.log = logging.getLogger('cma') CMAdb.debug = debug CMAdb.io = io from hbring import HbRing syslog = logging.handlers.SysLogHandler(address='/dev/log' , facility=logging.handlers.SysLogHandler.LOG_DAEMON) syslog.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s')) CMAdb.log.addHandler(syslog) CMAdb.log.setLevel(logging.DEBUG) url = ('http://%s:%d/db/data/' % (host, port)) #print >> sys.stderr, 'CREATING GraphDatabaseService("%s")' % url neodb = neo4j.GraphDatabaseService(url) self.db = neodb if cleanoutdb: CMAdb.log.info('Re-initializing the NEO4j database') self.delete_all() self.db = neodb CMAdb.use_network = use_network trycount=0 while True: try: CMAdb.cdb = CMAdb(db=neodb) # Neo4j started. All is well with the world. break except (RuntimeError, IOError, py2neo.exceptions.ClientError) as exc: print >> sys.stderr, 'TRYING AGAIN [[%s]...[%s]' % (url, str(exc)) trycount += 1 if trycount > retries: print >> sys.stderr, ('Neo4j still not started - giving up.') CMAdb.log.critical('Neo4j still not started - giving up.') raise RuntimeError('Neo4j not running - giving up [%s]' % str(exc)) if (trycount % 60) == 1: print >> sys.stderr, ('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) CMAdb.log.warning('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc))) # Let's try again in a second... time.sleep(1) Store.debug = debug Store.log = CMAdb.log CMAdb.store = Store(neodb, CMAconsts.uniqueindexes, CMAconsts.classkeymap , readonly=readonly) if not readonly: for classname in GraphNode.classmap: GraphNode.initclasstypeobj(CMAdb.store, classname) from transaction import Transaction CMAdb.transaction = Transaction(encryption_required=encryption_required) #print >> sys.stderr, 'CMAdb:', CMAdb #print >> sys.stderr, 'CMAdb.store(cmadb.py):', CMAdb.store CMAdb.TheOneRing = CMAdb.store.load_or_create(HbRing, name='The_One_Ring' , ringtype=HbRing.THEONERING) CMAdb.transaction.commit_trans(io) #print >> sys.stderr, 'COMMITTING Store' #print >> sys.stderr, 'Transaction Commit results:', CMAdb.store.commit() CMAdb.store.commit() #print >> sys.stderr, 'Store COMMITTED' else: CMAdb.transaction = None
def __init__(self, domain, monitorname, monitorclass, monitortype, interval , timeout, warntime=None, provider=None, arglist=None, argv=None): 'Create the requested monitoring rule object.' GraphNode.__init__(self, domain) self.monitorname = monitorname self.monitorclass = monitorclass self.monitortype = monitortype self.interval = int(interval) self.timeout = int(timeout) self.warntime = int(warntime) if warntime is not None else self.timeout/2 self.provider = provider self.isactive = False self.isworking = True self.reason = 'initial monitor creation' self.request_id = MonitorAction.request_id self.argv=argv MonitorAction.request_id += 1 if arglist is None: self.arglist = None self._arglist = {} elif isinstance(arglist, list): listlen = len(arglist) if (listlen % 2) != 0: raise(ValueError('arglist list must have an even number of elements')) self._arglist = {} for j in range(0, listlen, 2): self._arglist[arglist[j]] = arglist[j+1] self.arglist = arglist else: self._arglist = arglist self.arglist = [] for name in self._arglist: self.arglist.append(name) self.arglist.append(str(self._arglist[name]))
def __init__(self, MACaddr): GraphNode.__init__(self, domain='global') mac = pyNetAddr(MACaddr) if mac is None or mac.addrtype() != ADDR_FAMILY_802: raise ValueError('Not a legal MAC address [%s // %s]: %s (%s)' % (MACaddr, str(mac), str(mac.addrtype()), mac.addrlen())) self.MACaddr = str(mac)
def __init__( self, domain, monitorname, monitorclass, monitortype, interval, timeout, provider=None, arglist=None, argv=None ): "Create the requested monitoring rule object." GraphNode.__init__(self, domain) self.monitorname = monitorname self.monitorclass = monitorclass self.monitortype = monitortype self.interval = int(interval) self.timeout = int(timeout) self.provider = provider self.isactive = False self.isworking = True self.reason = "initial monitor creation" self.request_id = MonitorAction.request_id self.argv = argv MonitorAction.request_id += 1 if arglist is None: self.arglist = None self._arglist = {} elif isinstance(arglist, list): listlen = len(arglist) if (listlen % 2) != 0: raise (ValueError("arglist list must have an even number of elements")) self._arglist = {} for j in range(0, listlen, 2): self._arglist[arglist[j]] = arglist[j + 1] self.arglist = arglist else: self._arglist = arglist self.arglist = [] for name in self._arglist: self.arglist.append(name) self.arglist.append(str(self._arglist[name]))
def __init__(self, MACaddr): GraphNode.__init__(self, domain='global') mac = pyNetAddr(MACaddr) if mac is None or mac.addrtype() != ADDR_FAMILY_802: raise ValueError( 'Not a legal MAC address [%s // %s]: %s (%s)' % (MACaddr, str(mac), str(mac.addrtype()), mac.addrlen())) self.MACaddr = str(mac)
def __init__(self, firstname, lastname, dateofbirth=None): GraphNode.__init__(self, domain='global') self.firstname = firstname self.lastname = lastname if dateofbirth is not None: self.dateofbirth = dateofbirth else: self.dateofbirth = 'unknown'
def __init__(self, firstname, lastname, dateofbirth=None): GraphNode.__init__(self, domain='global') self.firstname = firstname self.lastname = lastname if dateofbirth is not None: self.dateofbirth = dateofbirth else: self.dateofbirth='unknown'
def __init__(self, domain, designation, roles=None): GraphNode.__init__(self, domain=domain) self.designation = str(designation).lower() self.monitors_activated = False if roles is None or roles == []: # Neo4j can't initialize node properties to empty arrays because # it wants to know what kind of array it is... roles = [''] self.roles = roles
def __init__(self, designation, domain='global', roles=None): GraphNode.__init__(self, domain) self.designation = designation.lower() if roles == None: roles = [''] self.roles = roles if domain is None: domain='global' self.domain = domain
def __init__(self, designation, domain='global', roles=None): GraphNode.__init__(self, domain) self.designation = designation.lower() if roles == None: roles = [''] self.roles = roles if domain is None: domain = 'global' self.domain = domain
def initstore(): global OurStore GraphNode.clean_graphnodes() if OurStore is not None: OurStore.clean_store() db.clear() CMAinit(None) OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap) CreateIndexes([cls.__name__ for cls in Classes]) return OurStore
def dbsetup(readonly=False, url=None): 'Set up our connection to Neo4j' if url is None: url = 'localhost.com:7474' Neo4jCreds().authenticate(url) ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={}) CMAinit(DummyIO(), readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) CMAdb.store = ourstore return ourstore
def __init__(self, name, ringtype): '''Constructor for a heartbeat ring. ''' GraphNode.__init__(self, domain=CMAdb.globaldomain) if ringtype < HbRing.SWITCH or ringtype > HbRing.THEONERING: raise ValueError("Invalid ring type [%s]" % str(ringtype)) self.ringtype = ringtype self.name = str(name) self.ourreltype = HbRing.memberprefix + self.name # Our membership relationship type self.ournexttype = HbRing.nextprefix + self.name # Our 'next' relationship type self._ringinitfinished = False self._insertpoint1 = None self._insertpoint2 = None
def initstore(): global version_printed db = neo4j.Graph(None) if not version_printed: print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version) print >> sys.stderr, 'USING py2neo VERSION %s' % str(py2neo.__version__) version_printed = True GraphNode.clean_graphnodes() db.delete_all() CMAinit(None) OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap) OurStore.clean_store() CreateIndexes(db, [cls.__name__ for cls in Classes]) return OurStore
def initstore(): global version_printed db = neo4j.Graph(None) if not version_printed: print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version) print >> sys.stderr, 'USING py2neo VERSION %s' % str( py2neo.__version__) version_printed = True GraphNode.clean_graphnodes() db.delete_all() CMAinit(None) OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap) OurStore.clean_store() CreateIndexes(db, [cls.__name__ for cls in Classes]) return OurStore
def __init__(self, queryname, JSON_metadata=None): '''Parameters ---------- JSON_metadata - a JSON string containing 'cypher': a string containing our cypher query 'descriptions': a dict containing descriptions in various languages 'language-key': A locale-language key 'short' a short description of this query in 'language-key' 'long' a long description of this query in 'language-key' 'parameters': a dict containing 'name': name of the parameter to the query 'type': the type of the parameter one of several noted 'min': optional minimum value for 'name' 'max': optional maximum value for 'name' 'enumlist': array of possible enum values (type = enum only) 'lang': language for this description as a dict: 'short':a short description of 'name' in language 'lang' 'long': a long description of 'name' in language 'lang' 'enumlist': dict of explanations for enumlist above Languages are the typical 'en' 'en_us', 'es', 'es_mx', as per the locale-usual Currently-known types are as follows: int - an integer float - an floating point number string - a string ipaddr - an IP address either ipv4 or ipv6 (as a string) macaddr - a 48 or 64 bit MAC address (as a string) bool - a boolean value hostname- a host name (as string - always converted to lower case) dnsname - a DNS name (as a string - always converted to lower case) enum - a finite enumeration of permissible values (case-insensitive) ''' GraphNode.__init__(self, domain='metadata') self.queryname = queryname self.JSON_metadata = JSON_metadata if JSON_metadata is None: self._JSON_metadata = None else: self._JSON_metadata = pyConfigContext(JSON_metadata) if self._JSON_metadata is None: raise ValueError('Parameter JSON_metadata is invalid [%s]' % JSON_metadata) self.validate_json() self._store = None self._db = None self._query = None
def post_db_init(self): GraphNode.post_db_init(self) if self._ringinitfinished: return self._ringinitfinished = True self._insertpoint1 = None self._insertpoint2 = None #print >> sys.stderr, 'CMAdb(hbring.py):', CMAdb #print >> sys.stderr, 'CMAdb.store(hbring.py):', CMAdb.store #print >> sys.stderr, 'Our relation type: %s' % self.ourreltype rellist = CMAdb.store.load_related(self, self.ourreltype, Drone) for rel in rellist: self._insertpoint1 = rel #print >> sys.stderr, 'INSERTPOINT1: ', self._insertpoint1 #print >> sys.stderr, 'Our relation type: %s' % self.ournexttype ip2rellist = CMAdb.store.load_related(self._insertpoint1, self.ournexttype, Drone) for rel2 in ip2rellist: self._insertpoint2 = rel2 break break
def post_db_init(self): GraphNode.post_db_init(self) if self._JSON_metadata is None: self._JSON_metadata = pyConfigContext(self.JSON_metadata) self.validate_json()
neodb = neo4j.GraphDatabaseService() neodb.clear() print >> sys.stderr, '========>classmap: %s' % (GraphNode.classmap) umap = {'ClientQuery': True} ckmap = { 'ClientQuery': { 'index': 'ClientQuery', 'kattr': 'queryname', 'value': 'None' } } qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" queries = ClientQuery.load_tree(qstore, "/home/alanr/monitor/src/queries") qlist = [q for q in queries] qstore.commit() print "%d node TREE LOADED!" % len(qlist) qe2 = qstore.load_or_create(ClientQuery, queryname='list') qe2.bind_store(qstore) testresult = '' for s in qe2.execute(None, idsonly=False, expandJSON=True): testresult += s print testresult # Test out a command line query for s in qe2.cmdline_exec(None): print s
} } } ''' q3 = ClientQuery('ipowners', metadata3) Neo4jCreds().authenticate() neodb = neo4j.Graph() neodb.delete_all() umap = {'ClientQuery': True} ckmap = {'ClientQuery': {'index': 'ClientQuery', 'kattr':'queryname', 'value':'None'}} qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(qstore, classname) print "LOADING TREE!" queries = ClientQuery.load_tree(qstore, "%s/../queries" % (os.path.dirname(sys.argv[0]))) qlist = [q for q in queries] qstore.commit() print "%d node TREE LOADED!" % len(qlist) qe2 = qstore.load_or_create(ClientQuery, queryname='list') qe2.bind_store(qstore) testresult = '' for s in qe2.execute(None, idsonly=False, expandJSON=True): testresult += s print 'RESULT', testresult # Test out a command line query for s in qe2.cmdline_exec(None):