def send_rule_event(oldstat, newstat, drone, ruleid, _ruleobj):
''' Newstat can never be None. '''
if oldstat is None:
if newstat == 'fail':
AssimEvent(drone,
AssimEvent.OBJWARN,
extrainfo={'ruleid': ruleid})
elif oldstat == 'pass':
if newstat == 'fail':
AssimEvent(drone,
AssimEvent.OBJWARN,
extrainfo={'ruleid': ruleid})
elif oldstat == 'fail':
if newstat == 'pass' or newstat == 'ignore' or newstat == 'NA':
AssimEvent(drone,
AssimEvent.OBJUNWARN,
extrainfo={'ruleid': ruleid})
elif oldstat == 'ignore':
if newstat == 'fail':
AssimEvent(drone,
AssimEvent.OBJWARN,
extrainfo={'ruleid': ruleid})
elif oldstat == 'NA':
if newstat == 'fail':
AssimEvent(drone,
AssimEvent.OBJWARN,
extrainfo={'ruleid': ruleid})
def __init__(self, constraints):
'''Initializer for AssimEventObserver class.
Parameters:
-----------
constraints: dict-like *or* callable() returning bool
A dict describing our desired events. The constraints in the dict are
effectively ANDed together. Each key is an attribute name in either
the event itself or its associated object. The value associated with
each attribute is either a list or a scalar value or something
implementing __contains__. A scalar value implies that it *must* have exactly
that value, otherwise it must be *in* the associated list/tuple/etc.
This should be able to constrain the type of event we're looking at, the
type of event-object we're looking at, and the domain of the event-object -
and lots of other potentially useful things.
See the "is_interesting" method below for implementation details...
If 'constraints' is a callable (that is, callable(constraints) is True), then
we will just call constraints(event) to see if the event is interesting to
this observer. Whatever 'constraints' returns will be interpreted in a
boolean context - so returning a bool would be a good idea...
'''
self.constraints = constraints
AssimEvent.registerobserver(self)
def ioerror(self, unusedevent):
'''This function gets called when we get an I/O error writing to the FIFO.
This is likely an EPIPE (broken pipe) error.
'''
unusedevent = unusedevent # Make pylint happy...
if self.maxerrcount is not None and self.errcount > self.maxerrcount:
AssimEvent.unregisterobserver(self)
def test_automonitor_strings_basic(self):
# Clean things out so we only see what we want to see...
AssimEvent.disable_all_observers()
ocf_string = '''{
# comment
"class": "ocf",
"type": "neo4j",
"provider": "assimilation",
"classconfig": [
[null, "@basename()", "java$"],
[null, "$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
["PORT", "serviceport", "[0-9]+$"],
["NEOHOME", "@argequals(-Dneo4j.home)", "/.*"]
]
}'''
ocf = MonitoringRule.ConstructFromString(ocf_string)
self.assertTrue(isinstance(ocf, OCFMonitoringRule))
lsb_string = '''{
# comment
"class": "lsb",
"type": "neo4j",
"classconfig": [
["@basename()", "java$"],
["$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
]
}'''
lsb = MonitoringRule.ConstructFromString(lsb_string)
self.assertTrue(isinstance(lsb, LSBMonitoringRule))
def ioerror(self, unusedevent):
'''This function gets called when we get an I/O error writing to the FIFO.
This is likely an EPIPE (broken pipe) error.
'''
unusedevent = unusedevent # Make pylint happy...
if self.maxerrcount is not None and self.errcount > self.maxerrcount:
AssimEvent.unregisterobserver(self)
def test_automonitor_strings_basic(self):
# Clean things out so we only see what we want to see...
AssimEvent.disable_all_observers()
ocf_string = '''{
# comment
"class": "ocf",
"type": "neo4j",
"provider": "assimilation",
"classconfig": [
[null, "@basename()", "java$"],
[null, "$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
["PORT", "serviceport", "[0-9]+$"],
["NEOHOME", "@argequals(-Dneo4j.home)", "/.*"]
]
}'''
ocf = MonitoringRule.ConstructFromString(ocf_string)
self.assertTrue(isinstance(ocf, OCFMonitoringRule))
lsb_string = '''{
# comment
"class": "lsb",
"type": "neo4j",
"classconfig": [
["@basename()", "java$"],
["$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
]
}'''
lsb = MonitoringRule.ConstructFromString(lsb_string)
self.assertTrue(isinstance(lsb, LSBMonitoringRule))
def test_automonitor_LSB_failures(self):
AssimEvent.disable_all_observers()
self.assertRaises(ValueError, LSBMonitoringRule, "neo4j-service", [])
self.assertRaises(ValueError, LSBMonitoringRule, "neo4j-service", (("a.b.c", ")"),))
self.assertRaises(ValueError, LSBMonitoringRule, "neo4j-service", ((1, 2, 3, 4, 5),))
self.assertRaises(ValueError, LSBMonitoringRule, "neo4j-service", ((1,),))
self.assertRaises(ValueError, LSBMonitoringRule, "neo4j-service", ((),))
def __init__(self, constraints):
'''Initializer for AssimEventObserver class.
Parameters:
-----------
constraints: dict-like *or* callable() returning bool
A dict describing our desired events. The constraints in the dict are
effectively ANDed together. Each key is an attribute name in either
the event itself or its associated object. The value associated with
each attribute is either a list or a scalar value or something
implementing __contains__. A scalar value implies that it *must* have exactly
that value, otherwise it must be *in* the associated list/tuple/etc.
This should be able to constrain the type of event we're looking at, the
type of event-object we're looking at, and the domain of the event-object -
and lots of other potentially useful things.
See the "is_interesting" method below for implementation details...
If 'constraints' is a callable (that is, callable(constraints) is True), then
we will just call constraints(event) to see if the event is interesting to
this observer. Whatever 'constraints' returns will be interpreted in a
boolean context - so returning a bool would be a good idea...
'''
self.constraints = constraints
AssimEvent.registerobserver(self)
def test_simple_init_bad(self):
'Perform a few simple AssimEvent bad initializations'
AssimEvent.observers = []
observer=DummyObserver()
badobserver=BadObserver()
AssimEvent.registerobserver(observer)
self.assertRaises(ValueError, AssimEvent, 'first', 999)
self.assertRaises(AttributeError, AssimEvent.registerobserver, badobserver)
def test_simple_init_bad(self):
'Perform a few simple AssimEvent bad initializations'
AssimEvent.enable_all_observers()
AssimEvent.observers = []
observer=DummyObserver()
badobserver=BadObserver()
AssimEvent.registerobserver(observer)
self.assertRaises(ValueError, AssimEvent, 'first', 999)
self.assertRaises(AttributeError, AssimEvent.registerobserver, badobserver)
def monitorchange(self, origaddr, monmsgobj):
'''
Make the necessary changes to the monitoring data when a particular
monitoring action changes status (to success or to failure)
Parameters
----------
origaddr: pyNetAddr
address where monitoring action originated
monmsgobj: pyConfigContext
object containing the monitoring message
'''
success = False
fubar = False
reason_enum = monmsgobj[REQREASONENUMNAMEFIELD]
if reason_enum == EXITED_ZERO:
success = True
explanation = 'is now operational'
elif reason_enum == EXITED_NONZERO:
explanation = 'monitoring failed with return code %s' % monmsgobj[
REQRCNAMEFIELD]
if REQSTRINGRETNAMEFIELD in monmsgobj:
explanation += ': %s' % str(monmsgobj[REQSTRINGRETNAMEFIELD])
elif reason_enum == EXITED_SIGNAL:
explanation = 'monitoring was killed by signal %s' % monmsgobj[
REQSIGNALNAMEFIELD]
elif reason_enum == EXITED_HUNG:
explanation = 'monitoring could not be killed'
elif reason_enum == EXITED_TIMEOUT:
explanation = 'monitoring timed out'
elif reason_enum == EXITED_INVAL:
explanation = 'invalid monitoring request'
else:
explanation = 'GOT REAL WEIRD (%d)' % int(reason_enum)
fubar = True
rscname = monmsgobj[CONFIGNAME_INSTANCE]
msg = 'Service %s %s' % (rscname, explanation)
self.isworking = success and not fubar
self.reason = explanation
print >> sys.stderr, 'MESSAGE:', msg
if fubar:
CMAdb.log.critical(msg)
else:
extrainfo = {
'comment': explanation,
'origaddr': origaddr,
'resourcename': rscname,
'monmsg': monmsgobj
}
if success:
CMAdb.log.info(msg)
AssimEvent(self, AssimEvent.OBJUP, extrainfo=extrainfo)
else:
CMAdb.log.warning(msg)
AssimEvent(self, AssimEvent.OBJDOWN, extrainfo=extrainfo)
def test_automonitor_LSB_failures(self):
AssimEvent.disable_all_observers()
self.assertRaises(ValueError, LSBMonitoringRule, 'neo4j-service', [])
self.assertRaises(ValueError, LSBMonitoringRule, 'neo4j-service',
(('a.b.c', ')'),))
self.assertRaises(ValueError, LSBMonitoringRule, 'neo4j-service',
((1,2,3,4,5),))
self.assertRaises(ValueError, LSBMonitoringRule, 'neo4j-service',
((1,),))
self.assertRaises(ValueError, LSBMonitoringRule, 'neo4j-service',
((),))
def test_fork_exec_event(self):
'''This test will create a fork/exec event observer script
and then test to see if its getting invoked properly...
'''
AssimEvent.enable_all_observers()
tmpdir = tempfile.mkdtemp('.d', 'testexec_')
(fd, pathname) = tempfile.mkstemp('.out.txt')
execscript = os.path.join(tmpdir, 'observer.sh')
makescript(execscript, pathname)
AssimEvent.observers = []
observer=ForkExecObserver(scriptdir=tmpdir)
dummyclient = ClientClass()
dummyclient.fred='fred'
dummyclient.sevenofnine='Annika'
dummyclient.foo = {'foo': 'bar'}
self.assertEqual(observer.listscripts(), [execscript,])
AssimEvent.registerobserver(observer)
AssimEvent(dummyclient, AssimEvent.CREATEOBJ)
AssimEvent(dummyclient, AssimEvent.OBJUP, extrainfo={'origaddr': '10.10.10.254'})
os.close(fd)
expectedcontent=\
'''====START====
ARG1=create
ARG2=ClientClass
ASSIM_fred=fred
ASSIM_nodetype=ClientClass
ASSIM_sevenofnine=Annika
==== JSON START ====
{"associatedobject":{"foo":{"foo":"bar"},"fred":"fred","nodetype":"ClientClass","sevenofnine":"Annika"},"eventtype":0,"extrainfo":null}
==== JSON END ====
====END====
====START====
ARG1=up
ARG2=ClientClass
ASSIM_fred=fred
ASSIM_nodetype=ClientClass
ASSIM_origaddr=10.10.10.254
ASSIM_sevenofnine=Annika
==== JSON START ====
{"associatedobject":{"foo":{"foo":"bar"},"fred":"fred","nodetype":"ClientClass","sevenofnine":"Annika"},"eventtype":1,"extrainfo":{"origaddr":"10.10.10.254"}}
==== JSON END ====
====END====
'''
TestAssimEvent.waitfor(pathname, expectedcontent)
f=open(pathname, 'r')
content=f.read()
f.close()
self.assertEqual(content, expectedcontent)
os.unlink(execscript)
os.unlink(pathname)
os.rmdir(tmpdir)
def test_fork_exec_killchild(self):
'''This test will create a fork/exec event observer script
and then kill the child listener and verify that it is handled
correctly.
'''
AssimEvent.enable_all_observers()
tmpdir = tempfile.mkdtemp('.d', 'testexec_')
(fd, pathname) = tempfile.mkstemp('.out.txt')
execscript = os.path.join(tmpdir, 'observer.sh')
makescript(execscript, pathname)
AssimEvent.observers = []
observer = ForkExecObserver(scriptdir=tmpdir)
dummyclient = ClientClass()
dummyclient.fred = 'fred'
dummyclient.sevenofnine = 'Annika'
dummyclient.foo = {'foo': 'bar'}
AssimEvent.registerobserver(observer)
try:
os.kill(observer.childpid, signal.SIGKILL)
except OSError:
# "docker build" doesn't let us kill processes...
# so we give up on this test and call it good...
os.unlink(execscript)
os.unlink(pathname)
os.rmdir(tmpdir)
return
time.sleep(0.1)
AssimEvent(dummyclient, AssimEvent.CREATEOBJ)
# Death of our FIFO child will cause it to get respawned, and
# message sent to new child. No messages should be lost.
expectedcontent=\
'''====START====
ARG1=create
ARG2=ClientClass
ASSIM_fred=fred
ASSIM_nodetype=ClientClass
ASSIM_sevenofnine=Annika
==== JSON START ====
{"associatedobject":{"foo":{"foo":"bar"},"fred":"fred","nodetype":"ClientClass","sevenofnine":"Annika"},"eventtype":0,"extrainfo":null}
==== JSON END ====
====END====
'''
TestAssimEvent.waitfor(pathname, expectedcontent)
f = open(pathname, 'r')
content = f.read()
f.close()
self.assertEqual(content, expectedcontent)
os.close(fd)
os.unlink(execscript)
os.unlink(pathname)
os.rmdir(tmpdir)
def test_fork_exec_killchild(self):
'''This test will create a fork/exec event observer script
and then kill the child listener and verify that it is handled
correctly.
'''
AssimEvent.enable_all_observers()
tmpdir = tempfile.mkdtemp('.d', 'testexec_')
(fd, pathname) = tempfile.mkstemp('.out.txt')
execscript = os.path.join(tmpdir, 'observer.sh')
makescript(execscript, pathname)
AssimEvent.observers = []
observer=ForkExecObserver(scriptdir=tmpdir)
dummyclient = ClientClass()
dummyclient.fred='fred'
dummyclient.sevenofnine='Annika'
dummyclient.foo = {'foo': 'bar'}
AssimEvent.registerobserver(observer)
try:
os.kill(observer.childpid, signal.SIGKILL)
except OSError:
# "docker build" doesn't let us kill processes...
# so we give up on this test and call it good...
os.unlink(execscript)
os.unlink(pathname)
os.rmdir(tmpdir)
return
time.sleep(0.1)
AssimEvent(dummyclient, AssimEvent.CREATEOBJ)
# Death of our FIFO child will cause it to get respawned, and
# message sent to new child. No messages should be lost.
expectedcontent=\
'''====START====
ARG1=create
ARG2=ClientClass
ASSIM_fred=fred
ASSIM_nodetype=ClientClass
ASSIM_sevenofnine=Annika
==== JSON START ====
{"associatedobject":{"foo":{"foo":"bar"},"fred":"fred","nodetype":"ClientClass","sevenofnine":"Annika"},"eventtype":0,"extrainfo":null}
==== JSON END ====
====END====
'''
TestAssimEvent.waitfor(pathname, expectedcontent)
f=open(pathname, 'r')
content=f.read()
f.close()
self.assertEqual(content, expectedcontent)
os.close(fd)
os.unlink(execscript)
os.unlink(pathname)
os.rmdir(tmpdir)
def test_eof(self):
'Get EOF with empty input'
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_test_eof()'
AssimEvent.disable_all_observers()
framesets=[]
io = TestIO(framesets, 0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
# just make sure it seems to do the right thing
(foo, bar) = io.recvframesets()
assert foo is None
del io
assert_no_dangling_Cclasses()
def test_eof(self):
'Get EOF with empty input'
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_test_eof()'
AssimEvent.disable_all_observers()
framesets=[]
io = TestIO(framesets, 0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
# just make sure it seems to do the right thing
(foo, bar) = io.recvframesets()
assert foo is None
del io
assert_no_dangling_Cclasses()
def dispatch(self, origaddr, frameset):
fstype = frameset.get_framesettype()
if CMAdb.debug:
CMAdb.log.debug(
"DispatchHBBACKALIVE: received [%s] FrameSet from address %s" %
(FrameSetTypes.get(fstype)[0], origaddr))
reporter = self.droneinfo.find(
origaddr) # System receiving the MARTIAN FrameSet
alivesrcaddr = None
for frame in frameset.iter():
frametype = frame.frametype()
if frametype == FrameTypes.IPPORT:
alivesrcaddr = frame.getnetaddr()
break
alivesrc = self.droneinfo.find(
alivesrcaddr) # Source of HBBACKALIVE event
if CMAdb.debug:
CMAdb.log.debug(
"DispatchHBBACKALIVE: received [%s] FrameSet from %s/%s about %s/%s"
% (FrameSetTypes.get(fstype)[0], reporter, origaddr, alivesrc,
alivesrcaddr))
if alivesrc.status != 'up':
if alivesrc.reason == 'HBSHUTDOWN':
# Just bad timing. All is well...
return
CMAdb.log.info(
'DispatchHBBACKALIVE: %s had been erroneously marked %s; reason %s'
% (alivesrc, alivesrc.status, alivesrc.reason))
alivesrc.status = 'up'
alivesrc.reason = 'HBBACKALIVE'
CMAdb.cdb.TheOneRing.join(alivesrc)
AssimEvent(alivesrc, AssimEvent.OBJUP)
def test_fork_exec_event(self):
'''This test will create a fork/exec event observer script
and then test to see if its getting invoked properly...
'''
tmpdir = tempfile.mkdtemp('.d', 'testexec_')
(fd, pathname) = tempfile.mkstemp('.out.txt')
execscript = os.path.join(tmpdir, 'observer.sh')
makescript(execscript, pathname)
AssimEvent.observers = []
observer=ForkExecObserver(scriptdir=tmpdir)
dummyclient = ClientClass()
dummyclient.fred='fred'
dummyclient.sevenofnine='Annika'
dummyclient.foo = {'foo': 'bar'}
self.assertEqual(observer.listscripts(), [execscript,])
AssimEvent.registerobserver(observer)
AssimEvent(dummyclient, AssimEvent.CREATEOBJ)
AssimEvent(dummyclient, AssimEvent.OBJUP, extrainfo={'origaddr': '10.10.10.254'})
os.close(fd)
expectedcontent=\
'''====START====
ARG1=create
ARG2=ClientClass
ASSIM_JSONobj={"associatedobject":{"foo":{"foo":"bar"},"fred":"fred","nodetype":"ClientClass","sevenofnine":"Annika"},"eventtype":0,"extrainfo":null}
ASSIM_fred=fred
ASSIM_nodetype=ClientClass
ASSIM_sevenofnine=Annika
====END====
====START====
ARG1=up
ARG2=ClientClass
ASSIM_JSONobj={"associatedobject":{"foo":{"foo":"bar"},"fred":"fred","nodetype":"ClientClass","sevenofnine":"Annika"},"eventtype":1,"extrainfo":{"origaddr":"10.10.10.254"}}
ASSIM_fred=fred
ASSIM_nodetype=ClientClass
ASSIM_origaddr=10.10.10.254
ASSIM_sevenofnine=Annika
====END====
'''
TestAssimEvent.waitfor(pathname, expectedcontent)
f=open(pathname, 'r')
content=f.read()
f.close()
self.assertEqual(content, expectedcontent)
os.unlink(execscript)
os.unlink(pathname)
os.rmdir(tmpdir)
def test_simple_init_good(self):
'Perform a few simple AssimEvent good initializations'
AssimEvent.enable_all_observers()
AssimEvent.observers = []
observer = DummyObserver()
AssimEvent.registerobserver(observer)
event1 = AssimEvent('first', AssimEvent.CREATEOBJ)
self.assertEqual(len(observer.events), 1)
self.assertTrue(observer.events[0], event1)
self.assertEqual(AssimEvent.unregisterobserver(observer), True)
event2 = AssimEvent('second', AssimEvent.CREATEOBJ)
self.assertEqual(len(observer.events), 1)
self.assertTrue(observer.events[0], event1)
AssimEvent.registerobserver(observer)
event3 = AssimEvent('third', AssimEvent.CREATEOBJ)
self.assertEqual(len(observer.events), 2)
self.assertTrue(observer.events[0], event3)
def send_rule_event(oldstat, newstat, drone, ruleid, ruleobj):
''' Newstat, ruleid, and ruleobj can never be None. '''
extrainfo = {'ruleid': ruleid, 'category': ruleobj[ruleid]['category']}
if oldstat is None:
if newstat == 'fail':
AssimEvent(drone, AssimEvent.OBJWARN, extrainfo=extrainfo)
elif oldstat == 'pass':
if newstat == 'fail':
AssimEvent(drone, AssimEvent.OBJWARN, extrainfo=extrainfo)
elif oldstat == 'fail':
if newstat == 'pass' or newstat == 'ignore' or newstat == 'NA':
AssimEvent(drone, AssimEvent.OBJUNWARN, extrainfo=extrainfo)
elif oldstat == 'ignore':
if newstat == 'fail':
AssimEvent(drone, AssimEvent.OBJWARN, extrainfo=extrainfo)
elif oldstat == 'NA':
if newstat == 'fail':
AssimEvent(drone, AssimEvent.OBJWARN, extrainfo=extrainfo)
def dispatch(self, origaddr, frameset):
fstype = frameset.get_framesettype()
if CMAdb.debug:
CMAdb.log.debug(
"DispatchHBMARTIAN: received [%s] FrameSet from address %s " %
(FrameSetTypes.get(fstype)[0], origaddr))
reporter = self.droneinfo.find(
origaddr) # System receiving the MARTIAN FrameSet
martiansrcaddr = None
for frame in frameset.iter():
frametype = frame.frametype()
if frametype == FrameTypes.IPPORT:
martiansrcaddr = frame.getnetaddr()
break
martiansrc = self.droneinfo.find(
martiansrcaddr) # Source of MARTIAN event
if CMAdb.debug:
CMAdb.log.debug(
"DispatchHBMARTIAN: received [%s] FrameSet from %s/%s about %s/%s"
% (FrameSetTypes.get(fstype)[0], reporter, origaddr,
martiansrc, martiansrcaddr))
if martiansrc.status != 'up':
if martiansrc.reason == 'HBSHUTDOWN':
# Just bad timing. All is well...
return
CMAdb.log.info(
'DispatchHBMARTIAN: %s had been erroneously marked %s; reason %s'
% (martiansrc, martiansrc.status, martiansrc.reason))
if CMAdb.debug:
CMAdb.log.info(
'DispatchHBMARTIAN: telling %s/%s to stop sending to %s/%s (%s case)'
% (martiansrc, martiansrcaddr, reporter, origaddr,
martiansrc.status))
martiansrc.status = 'up'
martiansrc.reason = 'HBMARTIAN'
martiansrc.send_hbmsg(martiansrcaddr,
FrameSetTypes.STOPSENDEXPECTHB, (origaddr, ))
CMAdb.cdb.TheOneRing.join(martiansrc)
AssimEvent(martiansrc, AssimEvent.OBJUP)
return
# OK, it's alive...
if CMAdb.cdb.TheOneRing.are_partners(reporter, martiansrc):
if CMAdb.debug:
CMAdb.log.debug(
'DispatchHBMARTIAN: Ignoring msg from %s about %s' %
(reporter, martiansrc))
else:
if CMAdb.debug:
CMAdb.log.info(
'DispatchHBMARTIAN: telling %s/%s to stop sending to %s/%s (%s case)'
% (martiansrc, martiansrcaddr, reporter, origaddr,
martiansrc.status))
# This probably isn't necessary in most cases, but it doesn't hurt anything.
# If the offender is just slow to update, he'll catch up...
martiansrc.send_hbmsg(martiansrcaddr,
FrameSetTypes.STOPSENDEXPECTHB, (origaddr, ))
def compute_score_updates(discovery_json, drone, rulesobj, newstats, oldstats):
'''We compute the score updates for the rules and results we've been given.
The drone is a Drone (or host), the 'rulesobj' contains the rules and their categories.
Statuses contains the results of evaluating the rules.
Our job is to compute the scores for each of the categories of rules in the
statuses, issue events for score changes, and update the category scores in the host.
We're storing the successes, failures, etc, for this discovery object for this drone.
Note that this can fail if we change our algorithm - because we don't know the values
the old algorithm gave us, only what the current algorithm gives us on the old results.
@TODO: We eventually want to update the scores for the domain to which this drone
belongs.
'''
_, oldcatscores, _ = BestPractices.compute_scores(drone, rulesobj, oldstats)
_, newcatscores, _ = BestPractices.compute_scores(drone, rulesobj, newstats)
keys = set(newcatscores)
keys |= set(oldcatscores)
# I have no idea why "keys = set(newcatscores) | set(oldcatscores)" did not work...
# It worked fine in an interactive python session...
diffs = {}
for category in keys:
newscore = newcatscores.get(category, 0.0)
oldscore = oldcatscores.get(category, 0.0)
catattr = Drone.bp_category_score_attrname(category)
# I just compare two floating point numbers without a lot of formality.
# This should be OK because they're both computed by the same algorithm
# And at this level algorithms mostly produce integers
# This is not a numerical analysis problem ;-)
if newscore != oldscore:
diff = newscore - oldscore
if category in diffs:
diffs[category] += diff
else:
diffs[category] = diff
eventtype = AssimEvent.OBJWARN if newscore > oldscore else AssimEvent.OBJUNWARN
extrainfo = {'category': category,
'oldscore': str(oldscore),
'newscore': str(newscore),
'discovery_type': discovery_json['discovertype'],
'discovery_description': discovery_json['description']
}
# POTENTIALCONCURRENCY
# As long as no one else is updating this attribute for this drone
# we shouldn't have concurrency problems.
oldval = getattr(drone, catattr) if hasattr(drone, catattr) else 0.0
setattr(drone, catattr, oldval + diff)
print >> sys.stderr, 'Setting %s.%s to %d' % (drone, catattr, oldval+diff)
AssimEvent(drone, eventtype, extrainfo=extrainfo)
return newcatscores, diffs
def test_get1pkt(self):
'Read a single packet'
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_test_eof()'
AssimEvent.disable_all_observers()
otherguy = pyNetAddr([1,2,3,4],)
strframe1=pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.")
fs = pyFrameSet(42)
fs.append(strframe1)
framesets=((otherguy, (strframe1,)),)
io = TestIO(framesets, 0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
gottenfs = io.recvframesets()
self.assertEqual(len(gottenfs), 2)
self.assertEqual(gottenfs, framesets[0])
gottenfs = io.recvframesets()
self.assertEqual(len(gottenfs), 2)
assert gottenfs[0] is None
io.cleanio()
del io
def __init__(self, constraints):
'''Initializer for AssimEventObserver class.
Parameters:
-----------
constraints: dict
A dict describing our desired events. The constraints in the dict are
effectively ANDed together. Each key is an attribute name in either
the event itself or its associated object. The value associated with
each attribute is either a list or a scalar value. A list implies that
any one of those values is acceptable. A scalar value implies that it
*must* have that value.
This should be able to constrain the type of event we're looking at, the
type of event-object we're looking at, and the domain of the event-object -
and lots of other potentially useful things.
See the "is_interesting" method below for implementation details...
'''
self.constraints = constraints
AssimEvent.registerobserver(self)
def __init__(self, constraints):
'''Initializer for AssimEventObserver class.
Parameters:
-----------
constraints: dict
A dict describing our desired events. The constraints in the dict are
effectively ANDed together. Each key is an attribute name in either
the event itself or its associated object. The value associated with
each attribute is either a list or a scalar value. A list implies that
any one of those values is acceptable. A scalar value implies that it
*must* have that value.
This should be able to constrain the type of event we're looking at, the
type of event-object we're looking at, and the domain of the event-object -
and lots of other potentially useful things.
See the "is_interesting" method below for implementation details...
'''
self.constraints = constraints
AssimEvent.registerobserver(self)
def test_get1pkt(self):
'Read a single packet'
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_test_eof()'
AssimEvent.disable_all_observers()
otherguy = pyNetAddr([1,2,3,4],)
strframe1=pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.")
fs = pyFrameSet(42)
fs.append(strframe1)
framesets=((otherguy, (strframe1,)),)
io = TestIO(framesets, 0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
gottenfs = io.recvframesets()
self.assertEqual(len(gottenfs), 2)
self.assertEqual(gottenfs, framesets[0])
gottenfs = io.recvframesets()
self.assertEqual(len(gottenfs), 2)
assert gottenfs[0] is None
io.cleanio()
del io
def test_echo1pkt(self):
'Read a packet and write it back out'
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_echo1pkt()'
AssimEvent.disable_all_observers()
strframe1=pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.")
fs = pyFrameSet(42)
fs.append(strframe1)
otherguy = pyNetAddr([1,2,3,4],)
framesets=((otherguy, (strframe1,)),)
io = TestIO(framesets, 0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
fslist = io.recvframesets() # read in a packet
self.assertEqual(len(fslist), 2)
self.assertEqual(fslist, framesets[0])
io.sendframesets(fslist[0], fslist[1]) # echo it back out
self.assertEqual(len(io.packetswritten), len(framesets))
gottenfs = io.recvframesets()
self.assertEqual(len(gottenfs), 2)
assert gottenfs[0] is None
io.cleanio()
del io
def test_echo1pkt(self):
'Read a packet and write it back out'
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_echo1pkt()'
AssimEvent.disable_all_observers()
strframe1=pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.")
fs = pyFrameSet(42)
fs.append(strframe1)
otherguy = pyNetAddr([1,2,3,4],)
framesets=((otherguy, (strframe1,)),)
io = TestIO(framesets, 0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
fslist = io.recvframesets() # read in a packet
self.assertEqual(len(fslist), 2)
self.assertEqual(fslist, framesets[0])
io.sendframesets(fslist[0], fslist[1]) # echo it back out
self.assertEqual(len(io.packetswritten), len(framesets))
gottenfs = io.recvframesets()
self.assertEqual(len(gottenfs), 2)
assert gottenfs[0] is None
io.cleanio()
del io
def compare_checksums(self, drone, oldobj, newobj):
'Compare checksums and complain about those that change'
designation = drone.designation
changes = {}
for oldfile in oldobj.keys():
if oldfile not in newobj:
continue
oldchecksum = oldobj[oldfile]
newchecksum = newobj[oldfile]
if oldchecksum == newchecksum:
continue
self.log.warning(
'On system %s: %s had checksum %s which is now %s' %
(designation, oldfile, oldchecksum, newchecksum))
changes[oldfile] = (oldchecksum, newchecksum)
extrainfo = {'CHANGETYPE': 'checksums', 'changes': changes}
AssimEvent(drone, AssimEvent.OBJUPDATE, extrainfo=extrainfo)
def test_simple_init_good(self):
'Perform a few simple AssimEvent good initializations'
AssimEvent.enable_all_observers()
AssimEvent.observers = []
observer=DummyObserver()
AssimEvent.registerobserver(observer)
event1 = AssimEvent('first', AssimEvent.CREATEOBJ)
self.assertEqual(len(observer.events), 1)
self.assertTrue(observer.events[0], event1)
self.assertEqual(AssimEvent.unregisterobserver(observer), True)
event2 = AssimEvent('second', AssimEvent.CREATEOBJ)
self.assertEqual(len(observer.events), 1)
self.assertTrue(observer.events[0], event1)
AssimEvent.registerobserver(observer)
event3 = AssimEvent('third', AssimEvent.CREATEOBJ)
self.assertEqual(len(observer.events), 2)
self.assertTrue(observer.events[0], event3)
def death_report(self, status, reason, fromaddr, frameset):
'Process a death/shutdown report for us. RIP us.'
from hbring import HbRing
frameset = frameset # We don't use the frameset at this point in time
if reason != 'HBSHUTDOWN':
if self.status != status or self.reason != reason:
CMAdb.log.info(
'Node %s has been reported as %s by address %s. Reason: %s'
% (self.designation, status, str(fromaddr), reason))
oldstatus = self.status
self.status = status
self.reason = reason
self.monitors_activated = False
self.time_status_ms = int(round(time.time() * 1000))
self.time_status_iso8601 = time.strftime('%Y-%m-%d %H:%M:%S')
if status == oldstatus:
# He was already dead, Jim.
return
# There is a need for us to be a little more sophisticated
# in terms of the number of peers this particular drone had
# It's here in this place that we will eventually add the ability
# to distinguish death of a switch or subnet or site from death of a single drone
for mightbering in CMAdb.store.load_in_related(self, None,
nodeconstructor):
if isinstance(mightbering, HbRing):
mightbering.leave(self)
deadip = pyNetAddr(self.select_ip(), port=self.port)
if CMAdb.debug:
CMAdb.log.debug('Closing connection to %s/%d' %
(deadip, DEFAULT_FSP_QID))
#
# So, if this is a death report from another system we could shut down ungracefully
# and it would be OK.
#
# But if it's a graceful shutdown, we need to not screw up the comm shutdown in progress
# If it's broken, our tests and the real world will eventually show that up :-D.
#
if reason != 'HBSHUTDOWN':
self._io.closeconn(DEFAULT_FSP_QID, deadip)
AssimEvent(self, AssimEvent.OBJDOWN)
def test_activate(self):
AssimEvent.disable_all_observers()
io = TestIO([],0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
dummy = CMAdb.store.load_or_create(MonitorAction, domain='global', monitorname='DummyName'
, monitorclass='OCF', monitortype='Dummy', interval=1, timeout=120, provider='heartbeat')
self.assertEqual(len(CMAdb.transaction.tree['packets']), 0)
CMAdb.store.commit()
CMAdb.transaction.commit_trans(io)
self.assertEqual(len(io.packetswritten), 0) # Shouldn't have sent out any pkts yet...
CMAdb.transaction = Transaction(encryption_required=False)
droneid = 1
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
droneAddr = pyNetAddr((127,0,0,1),1984)
droneone = CMAdb.store.load_or_create(Drone, designation=designation, port=1984
, startaddr=droneip, primary_ip_addr=droneip)
self.assertTrue(not dummy.isactive)
dummy.activate(droneone)
CMAdb.store.commit()
count=0
for obj in CMAdb.store.load_related(droneone, CMAconsts.REL_hosting, MonitorAction):
self.assertTrue(obj is dummy)
count += 1
self.assertEqual(count, 1)
self.assertTrue(dummy.isactive)
count=0
for obj in CMAdb.store.load_related(dummy, CMAconsts.REL_monitoring, Drone):
self.assertTrue(obj is droneone)
count += 1
self.assertEqual(count, 1)
#worked if we returned at or before here
CMAdb.transaction.commit_trans(io)
#failed if we return here or later
self.assertEqual(len(io.packetswritten), 1) # Did we send out exactly one packet?
if SavePackets:
#io.dumppackets()
for fstuple in io.packetswritten:
(dest, frameset) = fstuple
self.assertEqual(frameset.get_framesettype(), FrameSetTypes.DORSCOP)
for frame in frameset.iter():
self.assertEqual(frame.frametype(), FrameTypes.RSCJSON)
table = pyConfigContext(init=frame.getstr())
for field in ('class', 'type', 'instance', 'repeat'):
self.assertTrue(field in table)
if field == 'monitorclass' and table['monitorclass'] == 'OCF':
self.assertTrue('provider' in table)
for tup in (('class', str), ('type', str), ('resourcename', str)
, ('monitorclass', str), ('provider', str)
, ('repeat_interval', (int, long))
, ('timeout', (int,long))):
(n, t) = tup
if n in table:
self.assertTrue(isinstance(table[n], t))
# TODO: Add test for deactivating the resource(s)
io.cleanio()
del io
def test_automonitor_LSB_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({
'data': {
'lsb': {
'ssh',
'neo4j-service',
}
}
})
neoargs = (
('$argv[0]', r'.*/[^/]*java[^/]*$'), # Might be overkill
('$argv[3]', r'-server$'), # Probably overkill
('$argv[-1]', r'org\.neo4j\.server\.Bootstrapper$'),
)
neorule = LSBMonitoringRule('neo4j-service', neoargs)
sshnode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/sshd', ['/usr/bin/sshd', '-D' ]
#ProcessNode:
# (domain, host, nodename, pathname, argv, uid, gid, cwd, roles=None):
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
sshargs = (
# This means one of our nodes should have a value called
# pathname, and it should end in '/sshd'
('@basename()', 'sshd$'),
)
sshrule = LSBMonitoringRule('ssh', sshargs)
udevnode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/udevd', ['/usr/bin/udevd']
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
neoprocargs = ("/usr/bin/java", "-cp"
, "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/", "-server", "-XX:"
"+DisableExplicitGC"
, "-Dorg.neo4j.server.properties=conf/neo4j-server.properties"
, "-Djava.util.logging.config.file=conf/logging.properties"
, "-Dlog4j.configuration=file:conf/log4j.properties"
, "-XX:+UseConcMarkSweepGC"
, "-XX:+CMSClassUnloadingEnabled"
, "-Dneo4j.home=/var/lib/neo4j"
, "-Dneo4j.instance=/var/lib/neo4j"
, "-Dfile.encoding=UTF-8"
, "org.neo4j.server.Bootstrapper")
neonode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoprocargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
for tup in (sshrule.specmatch(ExpressionContext((udevnode, drone)))
, sshrule.specmatch(ExpressionContext((neonode, drone)))
, neorule.specmatch(ExpressionContext((sshnode, drone)))):
(prio, table) = tup
self.assertEqual(prio, MonitoringRule.NOMATCH)
self.assertTrue(table is None)
(prio, table) = sshrule.specmatch(ExpressionContext((sshnode, drone)))
self.assertEqual(prio, MonitoringRule.LOWPRIOMATCH)
self.assertEqual(table['monitorclass'], 'lsb')
self.assertEqual(table['monitortype'], 'ssh')
(prio, table) = neorule.specmatch(ExpressionContext((neonode, drone)))
self.assertEqual(prio, MonitoringRule.LOWPRIOMATCH)
self.assertEqual(table['monitorclass'], 'lsb')
self.assertEqual(table['monitortype'], 'neo4j-service')
if t2 < 10: t2 = 10
t3 = t2
if not DoAudit:
print >> sys.stderr, 'WARNING: Audits suppressed.'
if not doHBDEAD:
print >> sys.stderr, 'WARNING: Server death tests disabled.'
if not CheckForDanglingClasses:
print >> sys.stderr, 'WARNING: Memory Leak Detection disabled.'
elif not AssertOnDanglingClasses:
print >> sys.stderr, 'WARNING: Memory Leak assertions disabled (detection still enabled).'
#gc.set_threshold(t1, t2, t3)
AssimEvent.disable_all_observers()
def assert_no_dangling_Cclasses(doassert=None):
global CheckForDanglingClasses
global WorstDanglingCount
sys._clear_type_cache()
if doassert is None:
doassert = AssertOnDanglingClasses
CMAinit.uninit()
gc.collect() # For good measure...
count = proj_class_live_object_count()
#print >>sys.stderr, "CHECKING FOR DANGLING CLASSES (%d)..." % count
# Avoid cluttering the output up with redundant messages...
if count > WorstDanglingCount and CheckForDanglingClasses:
WorstDanglingCount = count
if doassert:
def test_automonitor_OCF_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({
'data': {
'ocf': {
'assimilation/neo4j',
}
}
})
kitchensink = OCFMonitoringRule('assimilation', 'neo4j',
( ('cantguess',) # length 1 - name
, ('port', '$port') # length 2 - name, expression
, (None, '$port') # length 2 - name, expression
, ('-', '$pathname') # length 2 - name, expression
, ('port', '$port', '[0-9]+$') # length 3 - name, expression, regex
, (None, '$pathname', '.*/java$') # length 3 - name, expression, regex
, (None, '@basename()', 'java$') # length 3 - name, expression, regex
, ('-', '$argv[-1]', r'org\.neo4j\.server\.Bootstrapper$')
# length 3 - name, expression, regex
, ('port', '@serviceport()', '[0-9]+$', re.I) # length 4 - name, expression, regex, flags
))
keys = kitchensink.nvpairs.keys()
keys.sort()
self.assertEqual(str(keys), "['cantguess', 'port']")
values = []
for key in keys:
values.append(kitchensink.nvpairs[key])
self.assertEqual(str(values), "[None, '@serviceport()']")
regex = re.compile('xxx')
regextype = type(regex)
exprlist = []
for tup in kitchensink._tuplespec:
self.assertEqual(type(tup[1]), regextype)
exprlist.append(tup[0])
self.assertEqual(str(exprlist)
, "['$port', '$pathname', '@basename()', '$argv[-1]', '@serviceport()']")
#
# That was a pain...
#
# Now, let's test the basics in a little more depth by creating what should be a working
# set of arguments to a (hypothetical) OCF resource agent
#
neo4j = OCFMonitoringRule('assimilation', 'neo4j',
( ('port', '$port')
, (None, '$pathname', '.*/java$')
, ('-', '$argv[-1]', r'org\.neo4j\.server\.Bootstrapper$')
, ('home', '@argequals(-Dneo4j.home)', '/.*')
, ('neo4j', '@basename(@argequals(-Dneo4j.home))', '.')
)
)
neoprocargs = ("/usr/bin/java", "-cp"
, "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/", "-server", "-XX:"
"+DisableExplicitGC"
, "-Dorg.neo4j.server.properties=conf/neo4j-server.properties"
, "-Djava.util.logging.config.file=conf/logging.properties"
, "-Dlog4j.configuration=file:conf/log4j.properties"
, "-XX:+UseConcMarkSweepGC"
, "-XX:+CMSClassUnloadingEnabled"
, "-Dneo4j.home=/var/lib/neo4j"
, "-Dneo4j.instance=/var/lib/neo4j"
, "-Dfile.encoding=UTF-8"
, "org.neo4j.server.Bootstrapper")
neonode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoprocargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
# We'll be missing the value of 'port'
neocontext = ExpressionContext((neonode, drone))
match = neo4j.specmatch(neocontext)
(prio, table, missing) = neo4j.specmatch(neocontext)
self.assertEqual(prio, MonitoringRule.PARTMATCH)
self.assertEqual(missing, ['port'])
# Now fill in the port value
neonode.port=7474
(prio, table) = neo4j.specmatch(neocontext)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(table['monitortype'], 'neo4j')
self.assertEqual(table['monitorclass'], 'ocf')
self.assertEqual(table['provider'], 'assimilation')
keys = table.keys()
keys.sort()
self.assertEqual(str(keys), "['arglist', 'monitorclass', 'monitortype', 'provider']")
arglist = table['arglist']
keys = arglist.keys()
keys.sort()
self.assertEqual(keys, ['home', 'neo4j', 'port'])
self.assertEqual(arglist['port'], '7474')
self.assertEqual(arglist['home'], '/var/lib/neo4j')
self.assertEqual(arglist['neo4j'], 'neo4j')
def test_automonitor_OCF_failures(self):
AssimEvent.disable_all_observers()
self.assertRaises(ValueError, OCFMonitoringRule, 'assimilation', 'neo4j',
((1,2,3,4,5),))
self.assertRaises(ValueError, OCFMonitoringRule, 'assimilation', 'neo4j',
((),))
def test_several_startups(self):
'''A very interesting test: We send a STARTUP message and get back a
SETCONFIG message and then send back a bunch of discovery requests.'''
if Store.debug:
raise ValueError('Debug enabled')
if DEBUG:
print >> sys.stderr, 'Running test_several_startups()'
AssimEvent.disable_all_observers()
OurAddr = pyNetAddr((10,10,10,5), 1984)
configinit = geninitconfig(OurAddr)
# Create the STARTUP FrameSets that our fake Drones should appear to send
fsin = []
droneid=0
for droneid in range(1,MaxDrone+1):
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
dronediscovery=hostdiscoveryinfo(droneid)
discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery)
fs = pyFrameSet(FrameSetTypes.STARTUP)
fs.append(designationframe)
fs.append(discoveryframe)
fsin.append((droneip, (fs,)))
addrone = droneipaddress(1)
maxdrones = droneid
if doHBDEAD:
# Create the HBDEAD FrameSets that our first fake Drone should appear to send
# concerning the death of its dearly departed peers
#print >> sys.stderr, 'KILLING THEM ALL!!!'
for droneid in range(2,maxdrones+1):
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
#deadframe=pyIpPortFrame(FrameTypes.IPPORT, addrstring=droneip)
fs = pyFrameSet(FrameSetTypes.HBSHUTDOWN)
#fs.append(deadframe)
hostframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
fs.append(hostframe)
fsin.append((droneip, (fs,)))
io = TestIO(fsin)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
assert CMAdb.io.config is not None
assimcli_check('loadqueries')
disp = MessageDispatcher( {
FrameSetTypes.STARTUP: DispatchSTARTUP(),
FrameSetTypes.HBDEAD: DispatchHBDEAD(),
FrameSetTypes.HBSHUTDOWN: DispatchHBSHUTDOWN(),
}, encryption_required=False)
config = pyConfigContext(init=configinit)
listener = PacketListener(config, disp, io=io, encryption_required=False)
io.mainloop = listener.mainloop
TestIO.mainloop = listener.mainloop
# We send the CMA a BUNCH of intial STARTUP packets
# and (optionally) a bunch of HBDEAD packets
assert CMAdb.io.config is not None
listener.listen()
# We audit after each packet is processed
# The auditing code will make sure all is well...
# But it doesn't know how many drones we just registered
Drones = CMAdb.store.load_cypher_nodes("START n=node:Drone('*:*') RETURN n", Drone)
Drones = [drone for drone in Drones]
#print >> sys.stderr, 'WE NOW HAVE THESE DRONES:', Drones
self.assertEqual(len(Drones), maxdrones)
if doHBDEAD:
# Verify that all drones except one are dead
#livecount, partnercount, ringmemberships
#self.check_live_counts(1, 0, 1)
assimcli_check("query allservers", maxdrones)
assimcli_check("query down", maxdrones-1)
assimcli_check("query crashed", 0)
assimcli_check("query shutdown", maxdrones-1)
else:
if maxdrones == 1:
partnercount=0
elif maxdrones == 2:
partnercount = 2
else:
partnercount=2*maxdrones
# livecount partnercount ringmemberships
#self.check_live_counts(maxdrones, partnercount, maxdrones)
assimcli_check("query allservers", maxdrones)
assimcli_check("query down", 0)
assimcli_check("query shutdown", 0)
assimcli_check("query unknownips", 0)
for droneid in range(1,MaxDrone+1):
droneip = droneipaddress(droneid)
assimcli_check("query findip %s" % str(droneip), 1)
if DoAudit:
auditalldrones()
auditallrings()
if DEBUG:
print "The CMA read %d packets." % io.packetsread
print "The CMA wrote %d packets." % io.writecount
#io.dumppackets()
io.config = None
io.cleanio()
del io
def test_automonitor_OCF_failures(self):
AssimEvent.disable_all_observers()
self.assertRaises(ValueError, OCFMonitoringRule, 'assimilation', 'neo4j',
((1,2,3,4,5),))
self.assertRaises(ValueError, OCFMonitoringRule, 'assimilation', 'neo4j',
((),))
def test_automonitor_OCF_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({
'data': {
'ocf': {
'assimilation/neo4j',
}
}
})
kitchensink = OCFMonitoringRule('assimilation', 'neo4j',
( ('cantguess',) # length 1 - name
, ('port', '$port') # length 2 - name, expression
, (None, '$port') # length 2 - name, expression
, ('-', '$pathname') # length 2 - name, expression
, ('port', '$port', '[0-9]+$') # length 3 - name, expression, regex
, (None, '$pathname', '.*/java$') # length 3 - name, expression, regex
, (None, '@basename()', 'java$') # length 3 - name, expression, regex
, ('-', '$argv[-1]', r'org\.neo4j\.server\.Bootstrapper$')
# length 3 - name, expression, regex
, ('port', '@serviceport()', '[0-9]+$', re.I) # length 4 - name, expression, regex, flags
))
keys = kitchensink.nvpairs.keys()
keys.sort()
self.assertEqual(str(keys), "['cantguess', 'port']")
values = []
for key in keys:
values.append(kitchensink.nvpairs[key])
self.assertEqual(str(values), "[None, '@serviceport()']")
regex = re.compile('xxx')
regextype = type(regex)
exprlist = []
for tup in kitchensink._tuplespec:
self.assertEqual(type(tup[1]), regextype)
exprlist.append(tup[0])
self.assertEqual(str(exprlist)
, "['$port', '$pathname', '@basename()', '$argv[-1]', '@serviceport()']")
#
# That was a pain...
#
# Now, let's test the basics in a little more depth by creating what should be a working
# set of arguments to a (hypothetical) OCF resource agent
#
neo4j = OCFMonitoringRule('assimilation', 'neo4j',
( ('port', '$port')
, (None, '$pathname', '.*/java$')
, ('-', '$argv[-1]', r'org\.neo4j\.server\.Bootstrapper$')
, ('home', '@argequals(-Dneo4j.home)', '/.*')
, ('neo4j', '@basename(@argequals(-Dneo4j.home))', '.')
)
)
neoprocargs = ("/usr/bin/java", "-cp"
, "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/", "-server", "-XX:"
"+DisableExplicitGC"
, "-Dorg.neo4j.server.properties=conf/neo4j-server.properties"
, "-Djava.util.logging.config.file=conf/logging.properties"
, "-Dlog4j.configuration=file:conf/log4j.properties"
, "-XX:+UseConcMarkSweepGC"
, "-XX:+CMSClassUnloadingEnabled"
, "-Dneo4j.home=/var/lib/neo4j"
, "-Dneo4j.instance=/var/lib/neo4j"
, "-Dfile.encoding=UTF-8"
, "org.neo4j.server.Bootstrapper")
neonode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoprocargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
# We'll be missing the value of 'port'
neocontext = ExpressionContext((neonode, drone))
match = neo4j.specmatch(neocontext)
(prio, table, missing) = neo4j.specmatch(neocontext)
self.assertEqual(prio, MonitoringRule.PARTMATCH)
self.assertEqual(missing, ['port'])
# Now fill in the port value
neonode.port=7474
(prio, table) = neo4j.specmatch(neocontext)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(table['monitortype'], 'neo4j')
self.assertEqual(table['monitorclass'], 'ocf')
self.assertEqual(table['provider'], 'assimilation')
keys = table.keys()
keys.sort()
self.assertEqual(str(keys), "['arglist', 'monitorclass', 'monitortype', 'provider']")
arglist = table['arglist']
keys = arglist.keys()
keys.sort()
self.assertEqual(keys, ['home', 'neo4j', 'port'])
self.assertEqual(arglist['port'], '7474')
self.assertEqual(arglist['home'], '/var/lib/neo4j')
self.assertEqual(arglist['neo4j'], 'neo4j')
if t2 < 10: t2 = 10
t3 = t2
if not DoAudit:
print >> sys.stderr, 'WARNING: Audits suppressed.'
if not doHBDEAD:
print >> sys.stderr, 'WARNING: Server death tests disabled.'
if not CheckForDanglingClasses:
print >> sys.stderr, 'WARNING: Memory Leak Detection disabled.'
elif not AssertOnDanglingClasses:
print >> sys.stderr, 'WARNING: Memory Leak assertions disabled (detection still enabled).'
#gc.set_threshold(t1, t2, t3)
AssimEvent.disable_all_observers()
def assert_no_dangling_Cclasses(doassert=None):
global CheckForDanglingClasses
global WorstDanglingCount
sys._clear_type_cache()
if doassert is None:
doassert = AssertOnDanglingClasses
CMAinit.uninit()
gc.collect() # For good measure...
count = proj_class_live_object_count()
#print >>sys.stderr, "CHECKING FOR DANGLING CLASSES (%d)..." % count
# Avoid cluttering the output up with redundant messages...
if count > WorstDanglingCount and CheckForDanglingClasses:
WorstDanglingCount = count
if doassert:
def test_automonitor_functions(self):
AssimEvent.disable_all_observers()
MonitoringRule.monitor_objects = {'service': {}, 'host':{}}
drone = FakeDrone({
'data': {
'ocf': {
'assimilation/neo4j',
},
'lsb': {
'bacula',
},
}
})
ocf_string = '''{
"class": "ocf", "type": "neo4j", "provider": "assimilation",
"classconfig": [
["classpath", "@flagvalue(-cp)"],
["ipaddr", "@serviceip($procinfo.listenaddrs)"],
["port", "@serviceport()", "[0-9]+$"]
]
}'''
ssh_json = '''{
"exe": "/usr/sbin/sshd",
"argv": [ "/usr/sbin/sshd", "-D" ],
"uid": "root",
"gid": "root",
"cwd": "/",
"listenaddrs": {
"0.0.0.0:22": {
"proto": "tcp",
"addr": "0.0.0.0",
"port": 22
},
":::22": {
"proto": "tcp6",
"addr": "::",
"port": 22
}
}
}'''
neo4j_json = '''{
"exe": "/usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java",
"argv": [ "/usr/bin/java", "-cp", "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar: ...", "-server", "-XX:+DisableExplicitGC", "-Dorg.neo4j.server.properties=conf/neo4
j-server.properties", "-Djava.util.logging.config.file=conf/logging.properties", "-Dlog4j.configuration=file:conf/log4j.properties", "-XX:
+UseConcMarkSweepGC", "-XX:+CMSClassUnloadingEnabled", "-Dneo4j.home=/var/lib/neo4j", "-Dneo4j.instance=/var/lib/neo4j", "-Dfile.encoding=
UTF-8", "org.neo4j.server.Bootstrapper" ],
"uid": "neo4j",
"gid": "neo4j",
"cwd": "/var/lib/neo4j",
"listenaddrs": {
":::1337": {
"proto": "tcp6",
"addr": "::",
"port": 1337
},
":::39185": {
"proto": "tcp6",
"addr": "::",
"port": 39185
}
}
}'''
bacula_json = '''{
"exe": "/usr/sbin/bacula-dir",
"argv": [ "/usr/sbin/bacula-dir", "-c", "/etc/bacula/bacula-dir.conf", "-u", "bacula", "-g", "bacula" ],
"uid": "bacula",
"gid": "bacula",
"cwd": "/",
"listenaddrs": {
"10.10.10.5:9101": {
"proto": "tcp",
"addr": "10.10.10.5",
"port": 9101
}
}
}'''
MonitoringRule.ConstructFromString(ocf_string)
neoargs = pyConfigContext(neo4j_json)['argv']
testnode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
testnode.procinfo = neo4j_json
context = ExpressionContext((testnode, drone))
(prio, match) = MonitoringRule.findbestmatch(context)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(match['arglist']['ipaddr'], '::1')
self.assertEqual(match['arglist']['port'], '1337')
testnode.procinfo = ssh_json
context = ExpressionContext((testnode, drone))
(prio, match) = MonitoringRule.findbestmatch(context)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(match['arglist']['port'], '22')
self.assertEqual(match['arglist']['ipaddr'], '127.0.0.1')
testnode.procinfo = bacula_json
context = ExpressionContext((testnode, drone))
(prio, match) = MonitoringRule.findbestmatch(context)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(match['arglist']['port'], '9101')
self.assertEqual(match['arglist']['ipaddr'], '10.10.10.5')
def test_automonitor_functions(self):
AssimEvent.disable_all_observers()
MonitoringRule.monitor_objects = {'service': {}, 'host':{}}
drone = FakeDrone({
'data': {
'ocf': {
'assimilation/neo4j',
},
'lsb': {
'bacula',
},
}
})
ocf_string = '''{
"class": "ocf", "type": "neo4j", "provider": "assimilation",
"classconfig": [
["classpath", "@flagvalue(-cp)"],
["ipaddr", "@serviceip($procinfo.listenaddrs)"],
["port", "@serviceport()", "[0-9]+$"]
]
}'''
ssh_json = '''{
"exe": "/usr/sbin/sshd",
"argv": [ "/usr/sbin/sshd", "-D" ],
"uid": "root",
"gid": "root",
"cwd": "/",
"listenaddrs": {
"0.0.0.0:22": {
"proto": "tcp",
"addr": "0.0.0.0",
"port": 22
},
":::22": {
"proto": "tcp6",
"addr": "::",
"port": 22
}
}
}'''
neo4j_json = '''{
"exe": "/usr/lib/jvm/java-7-openjdk-amd64/jre/bin/java",
"argv": [ "/usr/bin/java", "-cp", "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar: ...", "-server", "-XX:+DisableExplicitGC", "-Dorg.neo4j.server.properties=conf/neo4
j-server.properties", "-Djava.util.logging.config.file=conf/logging.properties", "-Dlog4j.configuration=file:conf/log4j.properties", "-XX:
+UseConcMarkSweepGC", "-XX:+CMSClassUnloadingEnabled", "-Dneo4j.home=/var/lib/neo4j", "-Dneo4j.instance=/var/lib/neo4j", "-Dfile.encoding=
UTF-8", "org.neo4j.server.Bootstrapper" ],
"uid": "neo4j",
"gid": "neo4j",
"cwd": "/var/lib/neo4j",
"listenaddrs": {
":::1337": {
"proto": "tcp6",
"addr": "::",
"port": 1337
},
":::39185": {
"proto": "tcp6",
"addr": "::",
"port": 39185
}
}
}'''
bacula_json = '''{
"exe": "/usr/sbin/bacula-dir",
"argv": [ "/usr/sbin/bacula-dir", "-c", "/etc/bacula/bacula-dir.conf", "-u", "bacula", "-g", "bacula" ],
"uid": "bacula",
"gid": "bacula",
"cwd": "/",
"listenaddrs": {
"10.10.10.5:9101": {
"proto": "tcp",
"addr": "10.10.10.5",
"port": 9101
}
}
}'''
MonitoringRule.ConstructFromString(ocf_string)
neoargs = pyConfigContext(neo4j_json)['argv']
testnode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
testnode.procinfo = neo4j_json
context = ExpressionContext((testnode, drone))
(prio, match) = MonitoringRule.findbestmatch(context)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(match['arglist']['ipaddr'], '::1')
self.assertEqual(match['arglist']['port'], '1337')
testnode.procinfo = ssh_json
context = ExpressionContext((testnode, drone))
(prio, match) = MonitoringRule.findbestmatch(context)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(match['arglist']['port'], '22')
self.assertEqual(match['arglist']['ipaddr'], '127.0.0.1')
testnode.procinfo = bacula_json
context = ExpressionContext((testnode, drone))
(prio, match) = MonitoringRule.findbestmatch(context)
self.assertEqual(prio, MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(match['arglist']['port'], '9101')
self.assertEqual(match['arglist']['ipaddr'], '10.10.10.5')
# You should have received a copy of the GNU General Public License
# along with the Assimilation Project software. If not, see http://www.gnu.org/licenses/
#
#
_suites = ['all', 'events']
import sys
sys.path.append("../cma")
sys.path.append("/usr/local/lib/python2.7/dist-packages")
from testify import *
import os, sys, tempfile, time, signal
from assimevent import AssimEvent
from assimeventobserver import ForkExecObserver
DEBUG=False
AssimEvent.enable_all_observers()
def makescript(createdscriptname, outfile):
'Create the requested script - outputting to requested file'
script='''#!/bin/sh
# Simple script to test external event interfaces
( echo "====START===="
j=1
for arg
do
echo "ARG${j}=$arg"
j=$(expr $j + 1)
done
env | grep '^ASSIM_' | LC_ALL=C sort
echo "==== JSON START ===="
cat
def test_startup(self):
'''A semi-interesting test: We send a STARTUP message and get back a
SETCONFIG message with lots of good stuff in it.
and for good measure, we also send along some discovery packets.
'''
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_startup()'
AssimEvent.disable_all_observers()
from dispatchtarget import DispatchTarget
droneid = 1
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
dronediscovery=hostdiscoveryinfo(droneid)
discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery)
fs = pyFrameSet(FrameSetTypes.STARTUP)
fs.append(designationframe)
fs.append(discoveryframe)
fs2 = pyFrameSet(FrameSetTypes.JSDISCOVERY)
osdiscovery=pyCstringFrame(FrameTypes.JSDISCOVER, self.OS_DISCOVERY)
fs2.append(osdiscovery)
fs3 = pyFrameSet(FrameSetTypes.JSDISCOVERY)
ulimitdiscovery=pyCstringFrame(FrameTypes.JSDISCOVER, self.ULIMIT_DISCOVERY)
fs3.append(ulimitdiscovery)
fsin = ((droneip, (fs,)), (droneip, (fs2,)), (droneip, (fs3,)))
io = TestIO(fsin,0)
#print >> sys.stderr, 'CMAinit: %s' % str(CMAinit)
#print >> sys.stderr, 'CMAinit.__init__: %s' % str(CMAinit.__init__)
OurAddr = pyNetAddr((127,0,0,1),1984)
configinit = geninitconfig(OurAddr)
config = pyConfigContext(init=configinit)
io.config = config
CMAinit(io, cleanoutdb=True, debug=DEBUG)
CMAdb.io.config = config
assimcli_check('loadqueries')
disp = MessageDispatcher(DispatchTarget.dispatchtable, encryption_required=False)
listener = PacketListener(config, disp, io=io, encryption_required=False)
io.mainloop = listener.mainloop
TestIO.mainloop = listener.mainloop
# We send the CMA an intial STARTUP packet
listener.listen()
# Let's see what happened...
#print >> sys.stderr, ('READ: %s' % io.packetsread)
#print >> sys.stderr, ('WRITTEN: %s' % len(io.packetswritten))
#print >> sys.stderr, ('PACKETS WRITTEN: %s' % str(io.packetswritten))
self.assertEqual(len(io.packetswritten), 2) # Did we send out four packets?
# Note that this change over time
# As we change discovery...
self.assertEqual(io.packetsread, 3) # Did we read 3 packets?
AUDITS().auditSETCONFIG(io.packetswritten[0], droneid, configinit)
assimcli_check("query allips", 1)
assimcli_check("query allservers", 1)
assimcli_check("query findip %s" % str(droneip), 1)
assimcli_check("query shutdown", 0)
assimcli_check("query crashed", 0)
assimcli_check("query unknownips", 0)
CMAdb.io.config = config
Drones = CMAdb.store.load_cypher_nodes("START n=node:Drone('*:*') RETURN n", Drone)
Drones = [drone for drone in Drones]
for drone in Drones:
self.check_discovery(drone, (dronediscovery, self.OS_DISCOVERY, self.ULIMIT_DISCOVERY))
self.assertEqual(len(Drones), 1) # Should only be one drone
io.config = None
io.cleanio()
del io
del ulimitdiscovery, osdiscovery, Drones
DispatchTarget.dispatchtable = {}
del DispatchTarget
def test_automonitor_LSB_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({"data": {"lsb": {"ssh", "neo4j-service"}}})
neoargs = (
("$argv[0]", r".*/[^/]*java[^/]*$"), # Might be overkill
("$argv[3]", r"-server$"), # Probably overkill
("$argv[-1]", r"org\.neo4j\.server\.Bootstrapper$"),
)
neorule = LSBMonitoringRule("neo4j-service", neoargs)
sshnode = ProcessNode(
"global",
"foofred",
"fred",
"/usr/bin/sshd",
["/usr/bin/sshd", "-D"]
# ProcessNode:
# (domain, host, nodename, pathname, argv, uid, gid, cwd, roles=None):
,
"root",
"root",
"/",
roles=(CMAconsts.ROLE_server,),
)
sshargs = (
# This means one of our nodes should have a value called
# pathname, and it should end in '/sshd'
("@basename()", "sshd$"),
)
sshrule = LSBMonitoringRule("ssh", sshargs)
udevnode = ProcessNode(
"global",
"foofred",
"fred",
"/usr/bin/udevd",
["/usr/bin/udevd"],
"root",
"root",
"/",
roles=(CMAconsts.ROLE_server,),
)
neoprocargs = (
"/usr/bin/java",
"-cp",
"/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/",
"-server",
"-XX:" "+DisableExplicitGC",
"-Dorg.neo4j.server.properties=conf/neo4j-server.properties",
"-Djava.util.logging.config.file=conf/logging.properties",
"-Dlog4j.configuration=file:conf/log4j.properties",
"-XX:+UseConcMarkSweepGC",
"-XX:+CMSClassUnloadingEnabled",
"-Dneo4j.home=/var/lib/neo4j",
"-Dneo4j.instance=/var/lib/neo4j",
"-Dfile.encoding=UTF-8",
"org.neo4j.server.Bootstrapper",
)
neonode = ProcessNode(
"global",
"foofred",
"fred",
"/usr/bin/java",
neoprocargs,
"root",
"root",
"/",
roles=(CMAconsts.ROLE_server,),
)
for tup in (
sshrule.specmatch(ExpressionContext((udevnode, drone))),
sshrule.specmatch(ExpressionContext((neonode, drone))),
neorule.specmatch(ExpressionContext((sshnode, drone))),
):
(prio, table) = tup
self.assertEqual(prio, MonitoringRule.NOMATCH)
self.assertTrue(table is None)
(prio, table) = sshrule.specmatch(ExpressionContext((sshnode, drone)))
self.assertEqual(prio, MonitoringRule.LOWPRIOMATCH)
self.assertEqual(table["monitorclass"], "lsb")
self.assertEqual(table["monitortype"], "ssh")
(prio, table) = neorule.specmatch(ExpressionContext((neonode, drone)))
self.assertEqual(prio, MonitoringRule.LOWPRIOMATCH)
self.assertEqual(table["monitorclass"], "lsb")
self.assertEqual(table["monitortype"], "neo4j-service")
def test_several_startups(self):
'''A very interesting test: We send a STARTUP message and get back a
SETCONFIG message and then send back a bunch of discovery requests.'''
if Store.debug:
raise ValueError('Debug enabled')
if DEBUG:
print >> sys.stderr, 'Running test_several_startups()'
AssimEvent.disable_all_observers()
OurAddr = pyNetAddr((10,10,10,5), 1984)
configinit = geninitconfig(OurAddr)
# Create the STARTUP FrameSets that our fake Drones should appear to send
fsin = []
droneid=0
for droneid in range(1,MaxDrone+1):
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
dronediscovery=hostdiscoveryinfo(droneid)
discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery)
fs = pyFrameSet(FrameSetTypes.STARTUP)
fs.append(designationframe)
fs.append(discoveryframe)
fsin.append((droneip, (fs,)))
addrone = droneipaddress(1)
maxdrones = droneid
if doHBDEAD:
# Create the HBDEAD FrameSets that our first fake Drone should appear to send
# concerning the death of its dearly departed peers
#print >> sys.stderr, 'KILLING THEM ALL!!!'
for droneid in range(2,maxdrones+1):
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
#deadframe=pyIpPortFrame(FrameTypes.IPPORT, addrstring=droneip)
fs = pyFrameSet(FrameSetTypes.HBSHUTDOWN)
#fs.append(deadframe)
hostframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
fs.append(hostframe)
fsin.append((droneip, (fs,)))
io = TestIO(fsin)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
assert CMAdb.io.config is not None
assimcli_check('loadqueries')
disp = MessageDispatcher( {
FrameSetTypes.STARTUP: DispatchSTARTUP(),
FrameSetTypes.HBDEAD: DispatchHBDEAD(),
FrameSetTypes.HBSHUTDOWN: DispatchHBSHUTDOWN(),
}, encryption_required=False)
config = pyConfigContext(init=configinit)
listener = PacketListener(config, disp, io=io, encryption_required=False)
io.mainloop = listener.mainloop
TestIO.mainloop = listener.mainloop
# We send the CMA a BUNCH of intial STARTUP packets
# and (optionally) a bunch of HBDEAD packets
assert CMAdb.io.config is not None
listener.listen()
# We audit after each packet is processed
# The auditing code will make sure all is well...
# But it doesn't know how many drones we just registered
Drones = CMAdb.store.load_cypher_nodes("START n=node:Drone('*:*') RETURN n", Drone)
Drones = [drone for drone in Drones]
#print >> sys.stderr, 'WE NOW HAVE THESE DRONES:', Drones
self.assertEqual(len(Drones), maxdrones)
if doHBDEAD:
# Verify that all drones except one are dead
#livecount, partnercount, ringmemberships
#self.check_live_counts(1, 0, 1)
assimcli_check("query allservers", maxdrones)
assimcli_check("query down", maxdrones-1)
assimcli_check("query crashed", 0)
assimcli_check("query shutdown", maxdrones-1)
else:
if maxdrones == 1:
partnercount=0
elif maxdrones == 2:
partnercount = 2
else:
partnercount=2*maxdrones
# livecount partnercount ringmemberships
#self.check_live_counts(maxdrones, partnercount, maxdrones)
assimcli_check("query allservers", maxdrones)
assimcli_check("query down", 0)
assimcli_check("query shutdown", 0)
assimcli_check("query unknownips", 0)
for droneid in range(1,MaxDrone+1):
droneip = droneipaddress(droneid)
assimcli_check("query findip %s" % str(droneip), 1)
if DoAudit:
auditalldrones()
auditallrings()
if DEBUG:
print "The CMA read %d packets." % io.packetsread
print "The CMA wrote %d packets." % io.writecount
#io.dumppackets()
io.config = None
io.cleanio()
del io
def test_activate(self):
AssimEvent.disable_all_observers()
io = TestIO([],0)
CMAinit(io, cleanoutdb=True, debug=DEBUG)
dummy = CMAdb.store.load_or_create(MonitorAction, domain='global', monitorname='DummyName'
, monitorclass='OCF', monitortype='Dummy', interval=1, timeout=120, provider='heartbeat')
self.assertEqual(len(CMAdb.transaction.tree['packets']), 0)
CMAdb.store.commit()
CMAdb.transaction.commit_trans(io)
self.assertEqual(len(io.packetswritten), 0) # Shouldn't have sent out any pkts yet...
CMAdb.transaction = Transaction(encryption_required=False)
droneid = 1
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
droneAddr = pyNetAddr((127,0,0,1),1984)
droneone = CMAdb.store.load_or_create(Drone, designation=designation, port=1984
, startaddr=droneip, primary_ip_addr=droneip)
self.assertTrue(not dummy.isactive)
dummy.activate(droneone)
CMAdb.store.commit()
count=0
for obj in CMAdb.store.load_related(droneone, CMAconsts.REL_hosting, MonitorAction):
self.assertTrue(obj is dummy)
count += 1
self.assertEqual(count, 1)
self.assertTrue(dummy.isactive)
count=0
for obj in CMAdb.store.load_related(dummy, CMAconsts.REL_monitoring, Drone):
self.assertTrue(obj is droneone)
count += 1
self.assertEqual(count, 1)
#worked if we returned at or before here
CMAdb.transaction.commit_trans(io)
#failed if we return here or later
self.assertEqual(len(io.packetswritten), 1) # Did we send out exactly one packet?
if SavePackets:
#io.dumppackets()
for fstuple in io.packetswritten:
(dest, frameset) = fstuple
self.assertEqual(frameset.get_framesettype(), FrameSetTypes.DORSCOP)
for frame in frameset.iter():
self.assertEqual(frame.frametype(), FrameTypes.RSCJSON)
table = pyConfigContext(init=frame.getstr())
for field in ('class', 'type', 'instance', 'repeat'):
self.assertTrue(field in table)
if field == 'monitorclass' and table['monitorclass'] == 'OCF':
self.assertTrue('provider' in table)
for tup in (('class', str), ('type', str), ('resourcename', str)
, ('monitorclass', str), ('provider', str)
, ('repeat_interval', (int, long))
, ('timeout', (int,long))):
(n, t) = tup
if n in table:
self.assertTrue(isinstance(table[n], t))
# TODO: Add test for deactivating the resource(s)
io.cleanio()
del io
def test_automonitor_LSB_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({
'data': {
'lsb': {
'ssh',
'neo4j-service',
}
}
})
neoargs = (
('$argv[0]', r'.*/[^/]*java[^/]*$'), # Might be overkill
('$argv[3]', r'-server$'), # Probably overkill
('$argv[-1]', r'org\.neo4j\.server\.Bootstrapper$'),
)
neorule = LSBMonitoringRule('neo4j-service', neoargs)
sshnode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/sshd', ['/usr/bin/sshd', '-D' ]
#ProcessNode:
# (domain, host, nodename, pathname, argv, uid, gid, cwd, roles=None):
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
sshargs = (
# This means one of our nodes should have a value called
# pathname, and it should end in '/sshd'
('@basename()', 'sshd$'),
)
sshrule = LSBMonitoringRule('ssh', sshargs)
udevnode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/udevd', ['/usr/bin/udevd']
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
neoprocargs = ("/usr/bin/java", "-cp"
, "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/", "-server", "-XX:"
"+DisableExplicitGC"
, "-Dorg.neo4j.server.properties=conf/neo4j-server.properties"
, "-Djava.util.logging.config.file=conf/logging.properties"
, "-Dlog4j.configuration=file:conf/log4j.properties"
, "-XX:+UseConcMarkSweepGC"
, "-XX:+CMSClassUnloadingEnabled"
, "-Dneo4j.home=/var/lib/neo4j"
, "-Dneo4j.instance=/var/lib/neo4j"
, "-Dfile.encoding=UTF-8"
, "org.neo4j.server.Bootstrapper")
neonode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoprocargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
for tup in (sshrule.specmatch(ExpressionContext((udevnode, drone)))
, sshrule.specmatch(ExpressionContext((neonode, drone)))
, neorule.specmatch(ExpressionContext((sshnode, drone)))):
(prio, table) = tup
self.assertEqual(prio, MonitoringRule.NOMATCH)
self.assertTrue(table is None)
(prio, table) = sshrule.specmatch(ExpressionContext((sshnode, drone)))
self.assertEqual(prio, MonitoringRule.LOWPRIOMATCH)
self.assertEqual(table['monitorclass'], 'lsb')
self.assertEqual(table['monitortype'], 'ssh')
(prio, table) = neorule.specmatch(ExpressionContext((neonode, drone)))
self.assertEqual(prio, MonitoringRule.LOWPRIOMATCH)
self.assertEqual(table['monitorclass'], 'lsb')
self.assertEqual(table['monitortype'], 'neo4j-service')
def dispatch(self, origaddr, frameset):
json = None
addrstr = repr(origaddr)
fstype = frameset.get_framesettype()
localtime = None
listenaddr = None
keyid = None
pubkey = None
keysize = None
#print >> sys.stderr, ("DispatchSTARTUP: received [%s] FrameSet from [%s]"
#% (FrameSetTypes.get(fstype)[0], addrstr))
if CMAdb.debug:
CMAdb.log.debug(
"DispatchSTARTUP: received [%s] FrameSet from [%s]" %
(FrameSetTypes.get(fstype)[0], addrstr))
if not self.io.connactive(origaddr):
self.io.closeconn(DEFAULT_FSP_QID, origaddr)
CMAdb.transaction.post_transaction_packets.append(
FrameSetTypes.ACKSTARTUP)
for frame in frameset.iter():
frametype = frame.frametype()
if frametype == FrameTypes.WALLCLOCK:
localtime = str(frame.getint())
elif frametype == FrameTypes.IPPORT:
listenaddr = frame.getnetaddr()
elif frametype == FrameTypes.HOSTNAME:
sysname = frame.getstr()
if sysname == CMAdb.nodename:
if origaddr.islocal():
CMAdb.log.info(
"Received STARTUP from local system (%s)" %
addrstr)
else:
addresses = ['127.0.0.1', '::ffff:127.0.0.1', '::1']
for address in addresses:
localhost = pyNetAddr(address)
self.io.addalias(localhost, origaddr)
CMAdb.log.info("Aliasing %s to %s" %
(localhost, origaddr))
elif frametype == FrameTypes.JSDISCOVER:
json = frame.getstr()
#print >> sys.stderr, 'GOT JSDISCOVER JSON: [%s] (strlen:%s,framelen:%s)' \
#% (json, len(json), frame.framelen())
elif frametype == FrameTypes.KEYID:
keyid = frame.getstr()
elif frametype == FrameTypes.PUBKEYCURVE25519:
pubkey = frame.framevalue()
keysize = frame.framelen()
joininfo = pyConfigContext(init=json)
origaddr, isNAT = self.validate_source_ip(sysname, origaddr, joininfo,
listenaddr)
CMAdb.log.info(
'Drone %s registered from address %s (%s) port %s, key_id %s' %
(sysname, origaddr, addrstr, origaddr.port(), keyid))
drone = self.droneinfo.add(sysname,
'STARTUP packet',
port=origaddr.port(),
primary_ip_addr=str(origaddr))
drone.listenaddr = str(listenaddr) # Seems good to hang onto this...
drone.isNAT = isNAT # ditto...
if CMAdb.debug:
CMAdb.log.debug('DRONE select_ip() result: %s' %
(drone.select_ip()))
CMAdb.log.debug('DRONE listenaddr: %s' % (drone.listenaddr))
CMAdb.log.debug('DRONE port: %s (%s)' %
(drone.port, type(drone.port)))
# Did they give us the crypto info we need?
if keyid is None or pubkey is None:
if CMAdb.debug:
CMAdb.log.debug(
'Drone %s registered with keyid %s and pubkey provided: %s'
% (self, keyid, pubkey is not None))
else:
if drone.key_id == '':
if not keyid.startswith(sysname + "@@"):
CMAdb.log.warning(
"Drone %s wants to register with key_id %s -- permitted.",
sysname, keyid)
if not cryptcurve25519_save_public_key(keyid, pubkey, keysize):
raise ValueError(
"Drone %s public key (key_id %s, %d bytes) is invalid."
% (sysname, keyid, keysize))
elif drone.key_id != keyid:
raise ValueError(
"Drone %s tried to register with key_id %s instead of %s."
% (sysname, keyid, drone.key_id))
drone.set_crypto_identity(keyid=keyid)
pyCryptFrame.dest_set_key_id(origaddr, keyid)
#
# THIS IS HERE BECAUSE OF A PROTOCOL BUG...
# @FIXME Protocol bug when starting up a connection if our first (this) packet gets lost,
# then the protocol doesn't retransmit it.
# More specifically, it seems to clear it out of the queue.
# This might be CMA bug or a protocol bug. It's not clear...
# The packet goes into the queue, but if that packet is lost in transmission, then when
# we come back around here, it's not in the queue any more, even though it
# definitely wasn't ACKed.
# Once this is fixed, this "add_packet" call needs to go *after* the 'if' statement below.
#
CMAdb.transaction.add_packet(origaddr, FrameSetTypes.SETCONFIG,
(str(self.config), ),
FrameTypes.CONFIGJSON)
if (localtime is not None):
if (drone.lastjoin == localtime):
CMAdb.log.warning('Drone %s [%s] sent duplicate STARTUP' %
(sysname, origaddr))
if CMAdb.debug:
self.io.log_conn(origaddr)
return
drone.lastjoin = localtime
#print >> sys.stderr, 'DRONE from find: ', drone, type(drone), drone.port
drone.startaddr = str(origaddr)
if json is not None:
drone.logjson(origaddr, json)
if CMAdb.debug:
CMAdb.log.debug('Joining TheOneRing: %s / %s / %s' %
(drone, type(drone), drone.port))
CMAdb.cdb.TheOneRing.join(drone)
if CMAdb.debug:
CMAdb.log.debug('Requesting Discovery from %s' % str(drone))
discovery_params = []
for agent in self.config['initial_discovery']:
params = ConfigFile.agent_params(self.config, 'discovery', agent,
sysname)
params['agent'] = agent
params['instance'] = '_init_%s' % agent
discovery_params.append(params)
# Discover the permissions of all the lists of files we're configured to ask about
# Note that there are several lists to keep the amount of data in any one list
# down to a somewhat more reasonable level. 'fileattrs' output is really verbose
for pathlist_name in self.config['perm_discovery_lists']:
paths = self.config[pathlist_name]
params = ConfigFile.agent_params(self.config, 'discovery',
'fileattrs', sysname)
params['agent'] = 'fileattrs'
params['instance'] = pathlist_name
params['parameters'] = {'ASSIM_filelist': paths}
discovery_params.append(params)
if CMAdb.debug:
CMAdb.log.debug('Discovery details: %s' % str(discovery_params))
for item in discovery_params:
CMAdb.log.debug('Discovery item details: %s' % str(item))
drone.request_discovery(discovery_params)
AssimEvent(drone, AssimEvent.OBJUP)
def test_automonitor_search_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({
'data': {
'ocf': {
'assimilation/neo4j',
},
'lsb': {
'neo4j-service',
}
}
})
MonitoringRule.monitor_objects = {'service': {}, 'host':{}}
ocf_string = '''{
"class": "ocf", "type": "neo4j", "provider": "assimilation",
"classconfig": [
[null, "@basename()", "java$"],
[null, "$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
["PORT", "$serviceport"],
["NEOHOME", "@argequals(-Dneo4j.home)", "/.*"]
]
}'''
MonitoringRule.ConstructFromString(ocf_string)
lsb_string = '''{
"class": "lsb", "type": "neo4j-service",
"classconfig": [
["@basename()", "java$"],
["$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
]
}'''
MonitoringRule.ConstructFromString(lsb_string)
neoprocargs = ("/usr/bin/java", "-cp"
, "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/", "-server", "-XX:"
"+DisableExplicitGC"
, "-Dneo4j.home=/var/lib/neo4j"
, "-Dneo4j.instance=/var/lib/neo4j"
, "-Dfile.encoding=UTF-8"
, "org.neo4j.server.Bootstrapper")
neonode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoprocargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
#neonode.serviceport=7474
context = ExpressionContext((neonode, drone))
first = MonitoringRule.findbestmatch(context)
second = MonitoringRule.findbestmatch(context, False)
list1 = MonitoringRule.findallmatches(context)
neonode.serviceport=7474
third = MonitoringRule.findbestmatch(context)
list2 = MonitoringRule.findallmatches(context)
# first should be the LSB instance
self.assertEqual(first[1]['monitorclass'], 'lsb')
self.assertEqual(first[0], MonitoringRule.LOWPRIOMATCH)
# second should be the incomplete OCF instance
self.assertEqual(second[1]['monitorclass'], 'ocf')
self.assertEqual(second[0], MonitoringRule.PARTMATCH)
# third should be the high priority OCF instance
self.assertEqual(third[1]['monitorclass'], 'ocf')
self.assertEqual(third[0], MonitoringRule.HIGHPRIOMATCH)
# list1 should be the incomplete OCF and the complete LSB - in that order
self.assertEqual(len(list1), 2)
# They should come out sorted by monitorclass
self.assertEqual(list1[0][0], MonitoringRule.LOWPRIOMATCH)
self.assertEqual(list1[0][1]['monitorclass'], 'lsb')
self.assertEqual(list1[1][0], MonitoringRule.PARTMATCH)
self.assertEqual(list1[1][1]['monitorclass'], 'ocf')
# third should be a complete OCF match
# list2 should be the complete OCF and the complete OCF - in that order
self.assertEqual(len(list2), 2)
self.assertEqual(list2[0][0], MonitoringRule.LOWPRIOMATCH)
self.assertEqual(list2[0][1]['monitorclass'], 'lsb')
self.assertEqual(list2[1][0], MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(list2[1][1]['monitorclass'], 'ocf')
def test_automonitor_search_basic(self):
AssimEvent.disable_all_observers()
drone = FakeDrone({
'data': {
'ocf': {
'assimilation/neo4j',
},
'lsb': {
'neo4j-service',
}
}
})
MonitoringRule.monitor_objects = {'service': {}, 'host':{}}
ocf_string = '''{
"class": "ocf", "type": "neo4j", "provider": "assimilation",
"classconfig": [
[null, "@basename()", "java$"],
[null, "$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
["PORT", "$serviceport"],
["NEOHOME", "@argequals(-Dneo4j.home)", "/.*"]
]
}'''
MonitoringRule.ConstructFromString(ocf_string)
lsb_string = '''{
"class": "lsb", "type": "neo4j-service",
"classconfig": [
["@basename()", "java$"],
["$argv[-1]", "org\\.neo4j\\.server\\.Bootstrapper$"],
]
}'''
MonitoringRule.ConstructFromString(lsb_string)
neoprocargs = ("/usr/bin/java", "-cp"
, "/var/lib/neo4j/lib/concurrentlinkedhashmap-lru-1.3.1.jar:"
"AND SO ON:"
"/var/lib/neo4j/system/lib/slf4j-api-1.6.2.jar:"
"/var/lib/neo4j/conf/", "-server", "-XX:"
"+DisableExplicitGC"
, "-Dneo4j.home=/var/lib/neo4j"
, "-Dneo4j.instance=/var/lib/neo4j"
, "-Dfile.encoding=UTF-8"
, "org.neo4j.server.Bootstrapper")
neonode = ProcessNode('global', 'foofred', 'fred', '/usr/bin/java', neoprocargs
, 'root', 'root', '/', roles=(CMAconsts.ROLE_server,))
#neonode.serviceport=7474
context = ExpressionContext((neonode, drone))
first = MonitoringRule.findbestmatch(context)
second = MonitoringRule.findbestmatch(context, False)
list1 = MonitoringRule.findallmatches(context)
neonode.serviceport=7474
third = MonitoringRule.findbestmatch(context)
list2 = MonitoringRule.findallmatches(context)
# first should be the LSB instance
self.assertEqual(first[1]['monitorclass'], 'lsb')
self.assertEqual(first[0], MonitoringRule.LOWPRIOMATCH)
# second should be the incomplete OCF instance
self.assertEqual(second[1]['monitorclass'], 'ocf')
self.assertEqual(second[0], MonitoringRule.PARTMATCH)
# third should be the high priority OCF instance
self.assertEqual(third[1]['monitorclass'], 'ocf')
self.assertEqual(third[0], MonitoringRule.HIGHPRIOMATCH)
# list1 should be the incomplete OCF and the complete LSB - in that order
self.assertEqual(len(list1), 2)
# They should come out sorted by monitorclass
self.assertEqual(list1[0][0], MonitoringRule.LOWPRIOMATCH)
self.assertEqual(list1[0][1]['monitorclass'], 'lsb')
self.assertEqual(list1[1][0], MonitoringRule.PARTMATCH)
self.assertEqual(list1[1][1]['monitorclass'], 'ocf')
# third should be a complete OCF match
# list2 should be the complete OCF and the complete OCF - in that order
self.assertEqual(len(list2), 2)
self.assertEqual(list2[0][0], MonitoringRule.LOWPRIOMATCH)
self.assertEqual(list2[0][1]['monitorclass'], 'lsb')
self.assertEqual(list2[1][0], MonitoringRule.HIGHPRIOMATCH)
self.assertEqual(list2[1][1]['monitorclass'], 'ocf')
def test_startup(self):
'''A semi-interesting test: We send a STARTUP message and get back a
SETCONFIG message with lots of good stuff in it.
and for good measure, we also send along some discovery packets.
'''
if BuildListOnly: return
if DEBUG:
print >> sys.stderr, 'Running test_startup()'
AssimEvent.disable_all_observers()
from dispatchtarget import DispatchTarget
droneid = 1
droneip = droneipaddress(droneid)
designation = dronedesignation(droneid)
designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation)
dronediscovery=hostdiscoveryinfo(droneid)
discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery)
fs = pyFrameSet(FrameSetTypes.STARTUP)
fs.append(designationframe)
fs.append(discoveryframe)
fs2 = pyFrameSet(FrameSetTypes.JSDISCOVERY)
osdiscovery=pyCstringFrame(FrameTypes.JSDISCOVER, self.OS_DISCOVERY)
fs2.append(osdiscovery)
fs3 = pyFrameSet(FrameSetTypes.JSDISCOVERY)
ulimitdiscovery=pyCstringFrame(FrameTypes.JSDISCOVER, self.ULIMIT_DISCOVERY)
fs3.append(ulimitdiscovery)
fsin = ((droneip, (fs,)), (droneip, (fs2,)), (droneip, (fs3,)))
io = TestIO(fsin,0)
#print >> sys.stderr, 'CMAinit: %s' % str(CMAinit)
#print >> sys.stderr, 'CMAinit.__init__: %s' % str(CMAinit.__init__)
OurAddr = pyNetAddr((127,0,0,1),1984)
configinit = geninitconfig(OurAddr)
config = pyConfigContext(init=configinit)
io.config = config
CMAinit(io, cleanoutdb=True, debug=DEBUG)
CMAdb.io.config = config
assimcli_check('loadqueries')
disp = MessageDispatcher(DispatchTarget.dispatchtable, encryption_required=False)
listener = PacketListener(config, disp, io=io, encryption_required=False)
io.mainloop = listener.mainloop
TestIO.mainloop = listener.mainloop
# We send the CMA an intial STARTUP packet
listener.listen()
# Let's see what happened...
#print >> sys.stderr, ('READ: %s' % io.packetsread)
#print >> sys.stderr, ('WRITTEN: %s' % len(io.packetswritten))
#print >> sys.stderr, ('PACKETS WRITTEN: %s' % str(io.packetswritten))
self.assertEqual(len(io.packetswritten), 2) # Did we send out four packets?
# Note that this change over time
# As we change discovery...
self.assertEqual(io.packetsread, 3) # Did we read 3 packets?
AUDITS().auditSETCONFIG(io.packetswritten[0], droneid, configinit)
assimcli_check("query allips", 1)
assimcli_check("query allservers", 1)
assimcli_check("query findip %s" % str(droneip), 1)
assimcli_check("query shutdown", 0)
assimcli_check("query crashed", 0)
assimcli_check("query unknownips", 0)
CMAdb.io.config = config
Drones = CMAdb.store.load_cypher_nodes("START n=node:Drone('*:*') RETURN n", Drone)
Drones = [drone for drone in Drones]
for drone in Drones:
self.check_discovery(drone, (dronediscovery, self.OS_DISCOVERY, self.ULIMIT_DISCOVERY))
self.assertEqual(len(Drones), 1) # Should only be one drone
io.config = None
io.cleanio()
del io
del ulimitdiscovery, osdiscovery, Drones
DispatchTarget.dispatchtable = {}
del DispatchTarget
