def __init__(self, ipaddr):
GraphNode.__init__(self, domain='global')
if isinstance(ipaddr, str):
ipaddr = pyNetAddr(ipaddr)
if isinstance(ipaddr, pyNetAddr):
ipaddr = ipaddr.toIPv6()
self.ipaddr = str(ipaddr)
def dbsetup(readonly=False):
'Set up our connection to Neo4j'
ourstore = Store(neo4j.GraphDatabaseService(), uniqueindexmap={}, classkeymap={})
CMAinit(None, readonly=readonly, use_network=False)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(ourstore, classname)
return ourstore
def dbsetup(readonly=False):
'Set up our connection to Neo4j'
ourstore = Store(neo4j.GraphDatabaseService(), uniqueindexmap={}, classkeymap={})
CMAinit(None, readonly=readonly, use_network=False)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(ourstore, classname)
return ourstore
def setup(dbhost='localhost', dbport=7474, dburl=None, querypath=None):
'''
Program to set up for running our REST server.
We do these things:
- Attach to the database
- Initialize our type objects so things like ClientQuery will work...
- Load the queries into the database from flat files
Not sure if doing this here makes the best sense, but it
works, and currently we're the only one who cares about them
so it seems reasonable -- at the moment ;-)
Also we're most likely to iterate changing on those relating to the
REST server, so fixing them just by restarting the REST server seems
to make a lot of sense (at the moment)
- Remember the set of queries in the 'allqueries' hash table
'''
if dburl is None:
dburl = ('http://%s:%d/db/data/' % (dbhost, dbport))
print >> sys.stderr, 'CREATING Graph("%s")' % dburl
neodb = neo4j.Graph(dburl)
qstore = Store(neodb, None, None)
print GraphNode.classmap
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(qstore, classname)
print "LOADING TREE!"
if querypath is None:
querypath = "/home/alanr/monitor/src/queries"
queries = ClientQuery.load_tree(qstore, querypath)
for q in queries:
allqueries[q.queryname] = q
qstore.commit()
for q in allqueries:
allqueries[q].bind_store(qstore)
def __init__(self, io, host='localhost', port=7474, cleanoutdb=False, debug=False
, retries=300, readonly=False, encryption_required=False, use_network=True):
'Initialize and construct a global database instance'
#print >> sys.stderr, 'CALLING NEW initglobal'
CMAdb.log = logging.getLogger('cma')
CMAdb.debug = debug
CMAdb.io = io
from hbring import HbRing
syslog = logging.handlers.SysLogHandler(address='/dev/log'
, facility=logging.handlers.SysLogHandler.LOG_DAEMON)
syslog.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
CMAdb.log.addHandler(syslog)
CMAdb.log.setLevel(logging.DEBUG)
url = ('http://%s:%d/db/data/' % (host, port))
#print >> sys.stderr, 'CREATING GraphDatabaseService("%s")' % url
neodb = neo4j.GraphDatabaseService(url)
self.db = neodb
if cleanoutdb:
CMAdb.log.info('Re-initializing the NEO4j database')
self.delete_all()
self.db = neodb
CMAdb.use_network = use_network
trycount=0
while True:
try:
CMAdb.cdb = CMAdb(db=neodb)
# Neo4j started. All is well with the world.
break
except (RuntimeError, IOError, py2neo.exceptions.ClientError) as exc:
print >> sys.stderr, 'TRYING AGAIN [[%s]...[%s]' % (url, str(exc))
trycount += 1
if trycount > retries:
print >> sys.stderr, ('Neo4j still not started - giving up.')
CMAdb.log.critical('Neo4j still not started - giving up.')
raise RuntimeError('Neo4j not running - giving up [%s]' % str(exc))
if (trycount % 60) == 1:
print >> sys.stderr, ('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc)))
CMAdb.log.warning('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc)))
# Let's try again in a second...
time.sleep(1)
Store.debug = debug
Store.log = CMAdb.log
CMAdb.store = Store(neodb, CMAconsts.uniqueindexes, CMAconsts.classkeymap
, readonly=readonly)
if not readonly:
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(CMAdb.store, classname)
from transaction import Transaction
CMAdb.transaction = Transaction(encryption_required=encryption_required)
#print >> sys.stderr, 'CMAdb:', CMAdb
#print >> sys.stderr, 'CMAdb.store(cmadb.py):', CMAdb.store
CMAdb.TheOneRing = CMAdb.store.load_or_create(HbRing, name='The_One_Ring'
, ringtype= HbRing.THEONERING)
CMAdb.transaction.commit_trans(io)
#print >> sys.stderr, 'COMMITTING Store'
#print >> sys.stderr, 'Transaction Commit results:', CMAdb.store.commit()
CMAdb.store.commit()
#print >> sys.stderr, 'Store COMMITTED'
else:
CMAdb.transaction = None
def __init__(self, io, host='localhost', port=7474, cleanoutdb=False, debug=False
, retries=300, readonly=False, encryption_required=False, use_network=True):
'Initialize and construct a global database instance'
#print >> sys.stderr, 'CALLING NEW initglobal'
CMAdb.log = logging.getLogger('cma')
CMAdb.debug = debug
CMAdb.io = io
from hbring import HbRing
syslog = logging.handlers.SysLogHandler(address='/dev/log'
, facility=logging.handlers.SysLogHandler.LOG_DAEMON)
syslog.setFormatter(logging.Formatter('%(name)s %(levelname)s: %(message)s'))
CMAdb.log.addHandler(syslog)
CMAdb.log.setLevel(logging.DEBUG)
url = ('http://%s:%d/db/data/' % (host, port))
#print >> sys.stderr, 'CREATING GraphDatabaseService("%s")' % url
neodb = neo4j.GraphDatabaseService(url)
self.db = neodb
if cleanoutdb:
CMAdb.log.info('Re-initializing the NEO4j database')
self.delete_all()
self.db = neodb
CMAdb.use_network = use_network
trycount=0
while True:
try:
CMAdb.cdb = CMAdb(db=neodb)
# Neo4j started. All is well with the world.
break
except (RuntimeError, IOError, py2neo.exceptions.ClientError) as exc:
print >> sys.stderr, 'TRYING AGAIN [[%s]...[%s]' % (url, str(exc))
trycount += 1
if trycount > retries:
print >> sys.stderr, ('Neo4j still not started - giving up.')
CMAdb.log.critical('Neo4j still not started - giving up.')
raise RuntimeError('Neo4j not running - giving up [%s]' % str(exc))
if (trycount % 60) == 1:
print >> sys.stderr, ('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc)))
CMAdb.log.warning('Waiting for Neo4j [%s] to start [%s].' % (url, str(exc)))
# Let's try again in a second...
time.sleep(1)
Store.debug = debug
Store.log = CMAdb.log
CMAdb.store = Store(neodb, CMAconsts.uniqueindexes, CMAconsts.classkeymap
, readonly=readonly)
if not readonly:
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(CMAdb.store, classname)
from transaction import Transaction
CMAdb.transaction = Transaction(encryption_required=encryption_required)
#print >> sys.stderr, 'CMAdb:', CMAdb
#print >> sys.stderr, 'CMAdb.store(cmadb.py):', CMAdb.store
CMAdb.TheOneRing = CMAdb.store.load_or_create(HbRing, name='The_One_Ring'
, ringtype=HbRing.THEONERING)
CMAdb.transaction.commit_trans(io)
#print >> sys.stderr, 'COMMITTING Store'
#print >> sys.stderr, 'Transaction Commit results:', CMAdb.store.commit()
CMAdb.store.commit()
#print >> sys.stderr, 'Store COMMITTED'
else:
CMAdb.transaction = None
def __init__(self, domain, monitorname, monitorclass, monitortype, interval
, timeout, warntime=None, provider=None, arglist=None, argv=None):
'Create the requested monitoring rule object.'
GraphNode.__init__(self, domain)
self.monitorname = monitorname
self.monitorclass = monitorclass
self.monitortype = monitortype
self.interval = int(interval)
self.timeout = int(timeout)
self.warntime = int(warntime) if warntime is not None else self.timeout/2
self.provider = provider
self.isactive = False
self.isworking = True
self.reason = 'initial monitor creation'
self.request_id = MonitorAction.request_id
self.argv=argv
MonitorAction.request_id += 1
if arglist is None:
self.arglist = None
self._arglist = {}
elif isinstance(arglist, list):
listlen = len(arglist)
if (listlen % 2) != 0:
raise(ValueError('arglist list must have an even number of elements'))
self._arglist = {}
for j in range(0, listlen, 2):
self._arglist[arglist[j]] = arglist[j+1]
self.arglist = arglist
else:
self._arglist = arglist
self.arglist = []
for name in self._arglist:
self.arglist.append(name)
self.arglist.append(str(self._arglist[name]))
def __init__(self, ipaddr):
GraphNode.__init__(self, domain='global')
if isinstance(ipaddr, str):
ipaddr = pyNetAddr(ipaddr)
if isinstance(ipaddr, pyNetAddr):
ipaddr = ipaddr.toIPv6()
self.ipaddr = str(ipaddr)
def __init__(self, MACaddr):
GraphNode.__init__(self, domain='global')
mac = pyNetAddr(MACaddr)
if mac is None or mac.addrtype() != ADDR_FAMILY_802:
raise ValueError('Not a legal MAC address [%s // %s]: %s (%s)'
% (MACaddr, str(mac), str(mac.addrtype()), mac.addrlen()))
self.MACaddr = str(mac)
def __init__(
self, domain, monitorname, monitorclass, monitortype, interval, timeout, provider=None, arglist=None, argv=None
):
"Create the requested monitoring rule object."
GraphNode.__init__(self, domain)
self.monitorname = monitorname
self.monitorclass = monitorclass
self.monitortype = monitortype
self.interval = int(interval)
self.timeout = int(timeout)
self.provider = provider
self.isactive = False
self.isworking = True
self.reason = "initial monitor creation"
self.request_id = MonitorAction.request_id
self.argv = argv
MonitorAction.request_id += 1
if arglist is None:
self.arglist = None
self._arglist = {}
elif isinstance(arglist, list):
listlen = len(arglist)
if (listlen % 2) != 0:
raise (ValueError("arglist list must have an even number of elements"))
self._arglist = {}
for j in range(0, listlen, 2):
self._arglist[arglist[j]] = arglist[j + 1]
self.arglist = arglist
else:
self._arglist = arglist
self.arglist = []
for name in self._arglist:
self.arglist.append(name)
self.arglist.append(str(self._arglist[name]))
def setup(dbhost='localhost', dbport=7474, dburl=None, querypath=None):
'''
Program to set up for running our REST server.
We do these things:
- Attach to the database
- Initialize our type objects so things like ClientQuery will work...
- Load the queries into the database from flat files
Not sure if doing this here makes the best sense, but it
works, and currently we're the only one who cares about them
so it seems reasonable -- at the moment ;-)
Also we're most likely to iterate changing on those relating to the
REST server, so fixing them just by restarting the REST server seems
to make a lot of sense (at the moment)
- Remember the set of queries in the 'allqueries' hash table
'''
if dburl is None:
dburl = ('http://%s:%d/db/data/' % (dbhost, dbport))
print >> sys.stderr, 'CREATING Graph("%s")' % dburl
neodb = neo4j.Graph(dburl)
qstore = Store(neodb, None, None)
print GraphNode.classmap
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(qstore, classname)
print "LOADING TREE!"
if querypath is None:
querypath = "/home/alanr/monitor/src/queries"
queries = ClientQuery.load_tree(qstore, querypath)
for q in queries:
allqueries[q.queryname] = q
qstore.commit()
for q in allqueries:
allqueries[q].bind_store(qstore)
def __init__(self, MACaddr):
GraphNode.__init__(self, domain='global')
mac = pyNetAddr(MACaddr)
if mac is None or mac.addrtype() != ADDR_FAMILY_802:
raise ValueError(
'Not a legal MAC address [%s // %s]: %s (%s)' %
(MACaddr, str(mac), str(mac.addrtype()), mac.addrlen()))
self.MACaddr = str(mac)
def __init__(self, firstname, lastname, dateofbirth=None):
GraphNode.__init__(self, domain='global')
self.firstname = firstname
self.lastname = lastname
if dateofbirth is not None:
self.dateofbirth = dateofbirth
else:
self.dateofbirth = 'unknown'
def __init__(self, firstname, lastname, dateofbirth=None):
GraphNode.__init__(self, domain='global')
self.firstname = firstname
self.lastname = lastname
if dateofbirth is not None:
self.dateofbirth = dateofbirth
else:
self.dateofbirth='unknown'
def __init__(self, domain, designation, roles=None):
GraphNode.__init__(self, domain=domain)
self.designation = str(designation).lower()
self.monitors_activated = False
if roles is None or roles == []:
# Neo4j can't initialize node properties to empty arrays because
# it wants to know what kind of array it is...
roles = ['']
self.roles = roles
def __init__(self, designation, domain='global', roles=None):
GraphNode.__init__(self, domain)
self.designation = designation.lower()
if roles == None:
roles = ['']
self.roles = roles
if domain is None:
domain='global'
self.domain = domain
def __init__(self, designation, domain='global', roles=None):
GraphNode.__init__(self, domain)
self.designation = designation.lower()
if roles == None:
roles = ['']
self.roles = roles
if domain is None:
domain = 'global'
self.domain = domain
def __init__(self, domain, designation, roles=None):
GraphNode.__init__(self, domain=domain)
self.designation = str(designation).lower()
self.monitors_activated = False
if roles is None or roles == []:
# Neo4j can't initialize node properties to empty arrays because
# it wants to know what kind of array it is...
roles = ['']
self.roles = roles
def initstore():
global OurStore
GraphNode.clean_graphnodes()
if OurStore is not None:
OurStore.clean_store()
db.clear()
CMAinit(None)
OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
CreateIndexes([cls.__name__ for cls in Classes])
return OurStore
def initstore():
global OurStore
GraphNode.clean_graphnodes()
if OurStore is not None:
OurStore.clean_store()
db.clear()
CMAinit(None)
OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
CreateIndexes([cls.__name__ for cls in Classes])
return OurStore
def dbsetup(readonly=False, url=None):
'Set up our connection to Neo4j'
if url is None:
url = 'localhost.com:7474'
Neo4jCreds().authenticate(url)
ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={})
CMAinit(DummyIO(), readonly=readonly, use_network=False)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(ourstore, classname)
CMAdb.store = ourstore
return ourstore
def dbsetup(readonly=False, url=None):
'Set up our connection to Neo4j'
if url is None:
url = 'localhost.com:7474'
Neo4jCreds().authenticate(url)
ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={})
CMAinit(DummyIO(), readonly=readonly, use_network=False)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(ourstore, classname)
CMAdb.store = ourstore
return ourstore
def __init__(self, name, ringtype):
'''Constructor for a heartbeat ring.
'''
GraphNode.__init__(self, domain=CMAdb.globaldomain)
if ringtype < HbRing.SWITCH or ringtype > HbRing.THEONERING:
raise ValueError("Invalid ring type [%s]" % str(ringtype))
self.ringtype = ringtype
self.name = str(name)
self.ourreltype = HbRing.memberprefix + self.name # Our membership relationship type
self.ournexttype = HbRing.nextprefix + self.name # Our 'next' relationship type
self._ringinitfinished = False
self._insertpoint1 = None
self._insertpoint2 = None
def initstore():
global version_printed
db = neo4j.Graph(None)
if not version_printed:
print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version)
print >> sys.stderr, 'USING py2neo VERSION %s' % str(py2neo.__version__)
version_printed = True
GraphNode.clean_graphnodes()
db.delete_all()
CMAinit(None)
OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
OurStore.clean_store()
CreateIndexes(db, [cls.__name__ for cls in Classes])
return OurStore
def initstore():
global version_printed
db = neo4j.Graph(None)
if not version_printed:
print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version)
print >> sys.stderr, 'USING py2neo VERSION %s' % str(
py2neo.__version__)
version_printed = True
GraphNode.clean_graphnodes()
db.delete_all()
CMAinit(None)
OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap)
OurStore.clean_store()
CreateIndexes(db, [cls.__name__ for cls in Classes])
return OurStore
def __init__(self, queryname, JSON_metadata=None):
'''Parameters
----------
JSON_metadata - a JSON string containing
'cypher': a string containing our cypher query
'descriptions': a dict containing descriptions in various languages
'language-key': A locale-language key
'short' a short description of this query in 'language-key'
'long' a long description of this query in 'language-key'
'parameters': a dict containing
'name': name of the parameter to the query
'type': the type of the parameter one of several noted
'min': optional minimum value for 'name'
'max': optional maximum value for 'name'
'enumlist': array of possible enum values (type = enum only)
'lang': language for this description as a dict:
'short':a short description of 'name' in language 'lang'
'long': a long description of 'name' in language 'lang'
'enumlist': dict of explanations for enumlist above
Languages are the typical 'en' 'en_us', 'es', 'es_mx', as per the locale-usual
Currently-known types are as follows:
int - an integer
float - an floating point number
string - a string
ipaddr - an IP address either ipv4 or ipv6 (as a string)
macaddr - a 48 or 64 bit MAC address (as a string)
bool - a boolean value
hostname- a host name (as string - always converted to lower case)
dnsname - a DNS name (as a string - always converted to lower case)
enum - a finite enumeration of permissible values (case-insensitive)
'''
GraphNode.__init__(self, domain='metadata')
self.queryname = queryname
self.JSON_metadata = JSON_metadata
if JSON_metadata is None:
self._JSON_metadata = None
else:
self._JSON_metadata = pyConfigContext(JSON_metadata)
if self._JSON_metadata is None:
raise ValueError('Parameter JSON_metadata is invalid [%s]' %
JSON_metadata)
self.validate_json()
self._store = None
self._db = None
self._query = None
def __init__(self, queryname, JSON_metadata=None):
'''Parameters
----------
JSON_metadata - a JSON string containing
'cypher': a string containing our cypher query
'descriptions': a dict containing descriptions in various languages
'language-key': A locale-language key
'short' a short description of this query in 'language-key'
'long' a long description of this query in 'language-key'
'parameters': a dict containing
'name': name of the parameter to the query
'type': the type of the parameter one of several noted
'min': optional minimum value for 'name'
'max': optional maximum value for 'name'
'enumlist': array of possible enum values (type = enum only)
'lang': language for this description as a dict:
'short':a short description of 'name' in language 'lang'
'long': a long description of 'name' in language 'lang'
'enumlist': dict of explanations for enumlist above
Languages are the typical 'en' 'en_us', 'es', 'es_mx', as per the locale-usual
Currently-known types are as follows:
int - an integer
float - an floating point number
string - a string
ipaddr - an IP address either ipv4 or ipv6 (as a string)
macaddr - a 48 or 64 bit MAC address (as a string)
bool - a boolean value
hostname- a host name (as string - always converted to lower case)
dnsname - a DNS name (as a string - always converted to lower case)
enum - a finite enumeration of permissible values (case-insensitive)
'''
GraphNode.__init__(self, domain='metadata')
self.queryname = queryname
self.JSON_metadata = JSON_metadata
if JSON_metadata is None:
self._JSON_metadata = None
else:
self._JSON_metadata = pyConfigContext(JSON_metadata)
if self._JSON_metadata is None:
raise ValueError('Parameter JSON_metadata is invalid [%s]' % JSON_metadata)
self.validate_json()
self._store = None
self._db = None
self._query = None
def post_db_init(self):
GraphNode.post_db_init(self)
if self._ringinitfinished:
return
self._ringinitfinished = True
self._insertpoint1 = None
self._insertpoint2 = None
#print >> sys.stderr, 'CMAdb(hbring.py):', CMAdb
#print >> sys.stderr, 'CMAdb.store(hbring.py):', CMAdb.store
#print >> sys.stderr, 'Our relation type: %s' % self.ourreltype
rellist = CMAdb.store.load_related(self, self.ourreltype, Drone)
for rel in rellist:
self._insertpoint1 = rel
#print >> sys.stderr, 'INSERTPOINT1: ', self._insertpoint1
#print >> sys.stderr, 'Our relation type: %s' % self.ournexttype
ip2rellist = CMAdb.store.load_related(self._insertpoint1, self.ournexttype, Drone)
for rel2 in ip2rellist:
self._insertpoint2 = rel2
break
break
def post_db_init(self):
GraphNode.post_db_init(self)
if self._JSON_metadata is None:
self._JSON_metadata = pyConfigContext(self.JSON_metadata)
self.validate_json()
neodb = neo4j.GraphDatabaseService()
neodb.clear()
print >> sys.stderr, '========>classmap: %s' % (GraphNode.classmap)
umap = {'ClientQuery': True}
ckmap = {
'ClientQuery': {
'index': 'ClientQuery',
'kattr': 'queryname',
'value': 'None'
}
}
qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(qstore, classname)
print "LOADING TREE!"
queries = ClientQuery.load_tree(qstore, "/home/alanr/monitor/src/queries")
qlist = [q for q in queries]
qstore.commit()
print "%d node TREE LOADED!" % len(qlist)
qe2 = qstore.load_or_create(ClientQuery, queryname='list')
qe2.bind_store(qstore)
testresult = ''
for s in qe2.execute(None, idsonly=False, expandJSON=True):
testresult += s
print testresult
# Test out a command line query
for s in qe2.cmdline_exec(None):
print s
}
}
}
'''
q3 = ClientQuery('ipowners', metadata3)
Neo4jCreds().authenticate()
neodb = neo4j.Graph()
neodb.delete_all()
umap = {'ClientQuery': True}
ckmap = {'ClientQuery': {'index': 'ClientQuery', 'kattr':'queryname', 'value':'None'}}
qstore = Store(neodb, uniqueindexmap=umap, classkeymap=ckmap)
for classname in GraphNode.classmap:
GraphNode.initclasstypeobj(qstore, classname)
print "LOADING TREE!"
queries = ClientQuery.load_tree(qstore, "%s/../queries" % (os.path.dirname(sys.argv[0])))
qlist = [q for q in queries]
qstore.commit()
print "%d node TREE LOADED!" % len(qlist)
qe2 = qstore.load_or_create(ClientQuery, queryname='list')
qe2.bind_store(qstore)
testresult = ''
for s in qe2.execute(None, idsonly=False, expandJSON=True):
testresult += s
print 'RESULT', testresult
# Test out a command line query
for s in qe2.cmdline_exec(None):
def post_db_init(self):
GraphNode.post_db_init(self)
if self._JSON_metadata is None:
self._JSON_metadata = pyConfigContext(self.JSON_metadata)
self.validate_json()