def processPRE(self, msglist, now):
passed = list()
# See what we can absorb
for msg in msglist:
if 'worm' in msg.dstdocks and 'data' in msg.dstgroups:
ifin = msg._receivedon
if ifin not in self.blobs:
when = now + 0.1
self.blobs[ifin] = (when, list())
self.msgintf.needPush('PRE', when) # if past a current active push request, the earliest request wins
self.blobs[ifin][1].append(msg.data)
else:
passed.append(msg) # regular messages
# See what now needs to be sent
for transport, (timer, blob) in self.blobs.items():
if timer > now:
self.msgintf.needPush('PRE', timer) # make sure we get later pushes as needed, again min(...) wins
continue
del self.blobs[transport] # pull out entry
if len(blob) > 0:
log.info("Sending collected worm messages (%s)", len(blob))
msg = MAGIMessage(groups='data', docks='worm', contenttype=MAGIMessage.TEXT, data='\n'.join(blob))
self.nameid.processOUT([msg]) # need a new source and valid msg ID
msg._receivedon = transport # lets routing know where NOT to send it again, no need to smurf ourselves
passed.append(msg)
return passed
def groupRouterMessage(self, msg):
log.debug("Processing group router msg: %s", msg)
try:
tgl = self.transportGroupLists[msg._receivedon.fileno()]
request = yaml.load(msg.data)
response = None
if 'resend' in request:
response = {
'set': list(tgl.rxGroups),
'count': len(tgl.rxGroups),
'checksum': listChecksum(tgl.rxGroups)
}
listmsg = MAGIMessage(contenttype=MAGIMessage.YAML, nodes=[msg.src], docks=[GroupRouter.DOCK], data=yaml.safe_dump(response))
listmsg._routed = [msg._receivedon.fileno()]
self.msgintf.send(listmsg)
else:
# Discover what actually changed
(added, deleted) = tgl.processMessage(msg.src, request)
# Pass changes onto other transport for processing
for fd, othertgl in self.transportGroupLists.iteritems():
if othertgl is tgl: continue
othertgl.requestChanges(added, deleted)
except Exception, e:
log.error("Failed to process router message: %s", e, exc_info=1)
def test_Record(self):
""" Test basic recording of defaults """
ip = testbed.getInterfaceList()[0].ip
req = {
'version': 1.0,
'method':'setConfiguration',
'args': {
'interval': 2,
'filters': {
'cnt1': { 'input': ip },
'cnt2': { 'output': ip },
}
}
}
start = {
'version': 1.0,
'method': 'startCollection',
}
stop = {
'version': 1.0,
'method': 'stopCollection',
}
self.fixture.inject(MAGIMessage(src='guinode', data=yaml.safe_dump(req)))
self.fixture.inject(MAGIMessage(src='guinode', data=yaml.safe_dump(start)))
time.sleep(5)
self.fixture.inject(MAGIMessage(src='guinode', data=yaml.safe_dump(stop)))
time.sleep(0.1) # let it process it first
def requestChanges(self, added, deleted):
"""
Caller is letting us know that another transport has changed the list of groups it wishes to know about.
This method will determine what message it needs to send (if any) out the associated transport
"""
if self.fileno == 0:
return # shortcut for localhost
# Determine what addition message to send, if any
request = dict()
additions = list()
for g in added:
if self.rxGroups.inc(g):
additions.append(g) # This is actually new to us
if len(additions) > 0:
request['add'] = additions
# Determine what removal message to send, if any
deletions = list()
for g in deleted:
if self.rxGroups.dec(g):
deletions.append(g) # Noone appears to want this anymore
if len(deletions) > 0:
request['del'] = deletions
# If something did change send out the updates
if len(additions) > 0 or len(deletions) > 0:
request['count'] = len(self.rxGroups)
request['checksum'] = listChecksum(self.rxGroups)
msg = MAGIMessage(contenttype=MAGIMessage.YAML, groups=[GroupRouter.ONEHOPNODES], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request))
log.debug("Sending request for group changes to neighbors: %s", msg)
msg._routed = [self.fileno]
self.msgintf.send(msg)
def doOneRequest(self, request, listeningDock, needTrigger):
# load agent request
self.q.inject(MAGIMessage(docks='daemon',
data=yaml.safe_dump(request)))
self._waitForListen(listeningDock)
if needTrigger:
transmitted = self.q.extract(True, 2)
self.assertIn('control', transmitted.msg.dstdocks)
self.assertIn('control', transmitted.msg.dstgroups)
# single test message
self.q.inject(MAGIMessage(docks=listeningDock, data='testdata'))
transmitted = self.q.extract(True, 2)
self.assertEquals('hello', transmitted.msg.dstdocks.pop())
self.assertEquals(True, transmitted.args['acknowledgement'])
self.assertEquals(98765, transmitted.args['timestamp'])
def run(self):
msg = self.messenger.next()
if msg.data == 'testdata':
self.messenger.send(MAGIMessage(
nodes='somewhere',
docks='hello',
data="response from testAgentThread"),
acknowledgement=True,
timestamp=98765)
def getMessages(self):
call = {'version': 1.0, 'method': self.method, 'args': self.args}
if self.trigger:
call['trigger'] = self.trigger
return [
MAGIMessage(groups=self.groups,
nodes=self.nodes,
docks=self.docks,
contenttype=MAGIMessage.YAML,
data=yaml.dump(call, width=10000))
]
def test_Alive(self):
""" Test of alive ping request """
self.q.inject(
MAGIMessage(src='othernode',
docks='daemon',
data=yaml.safe_dump({
'version': 1.0,
'method': 'ping'
})))
self.assertEquals('pong', self.q.extract(True, 2).msg.dstdocks.pop())
time.sleep(0.2)
def processMessage(self, src, request):
"""
Processes incoming routing messages, determine what has changed in our state and return that. An error response may be
sent to the sender if a synchronization problem is detected.
"""
# Update groups for this src, recreate the full set
if src not in self.srccache:
self.srccache[src] = NeighborGroupList()
nentry = self.srccache[src]
tadded = list()
tremoved = list()
try:
if 'add' in request:
added = nentry.add(request['count'], request['checksum'], set(request['add']))
tadded = self.txGroups.incGroup(added)
if 'del' in request:
removed = nentry.remove(request['count'], request['checksum'], set(request['del']))
tremoved = self.txGroups.decGroup(removed)
if 'set' in request:
if 'add' in request or 'del' in request:
log.error("Got a group route message with a bad set of requests (%s)", request.keys())
else:
if request['count'] == len(nentry.nodeGroups) and request['checksum'] == nentry.checksum:
return ([], []) # Nothing actually changed according to our list, shortcut it here
(added, removed) = nentry.newlist(request['count'], request['checksum'], set(request['set']))
tadded = self.txGroups.incGroup(added)
tremoved = self.txGroups.decGroup(removed)
except GroupStateError:
msg = MAGIMessage(contenttype=MAGIMessage.YAML, nodes=[src], docks=[GroupRouter.DOCK], data=yaml.safe_dump({'resend':True}))
msg._routed = [self.fileno]
self.msgintf.send(msg)
raise
# Return the lists that we changed
return (tadded, tremoved)
def test_Joiner(self):
""" Test of join and leave requests """
request = {
'version': 1.0,
'method': 'joinGroup',
'args': {
'group': 'supergroup',
'nodes': ['mynode', 'n1', 'n2']
}
}
self.q.inject(MAGIMessage(docks='daemon',
data=yaml.safe_dump(request)))
time.sleep(0.1)
self.assert_(self.q.checkMembership('supergroup'))
request['method'] = 'leaveGroup'
self.q.inject(MAGIMessage(docks='daemon',
data=yaml.safe_dump(request)))
time.sleep(0.1)
self.assert_(not self.q.checkMembership('supergroup'))
def processPRE(self, msglist, now):
passed = list()
# See what we can absorb
for msg in msglist:
if 'worm' in msg.dstdocks and 'data' in msg.dstgroups:
ifin = msg._receivedon
if ifin not in self.blobs:
when = now + 0.1
self.blobs[ifin] = (when, list())
self.msgintf.needPush(
'PRE', when
) # if past a current active push request, the earliest request wins
self.blobs[ifin][1].append(msg.data)
else:
passed.append(msg) # regular messages
# See what now needs to be sent
for transport, (timer, blob) in self.blobs.items():
if timer > now:
self.msgintf.needPush(
'PRE', timer
) # make sure we get later pushes as needed, again min(...) wins
continue
del self.blobs[transport] # pull out entry
if len(blob) > 0:
log.info("Sending collected worm messages (%s)", len(blob))
msg = MAGIMessage(groups='data',
docks='worm',
contenttype=MAGIMessage.TEXT,
data='\n'.join(blob))
self.nameid.processOUT(
[msg]) # need a new source and valid msg ID
msg._receivedon = transport # lets routing know where NOT to send it again, no need to smurf ourselves
passed.append(msg)
return passed
def test_apache(self):
""" Test load, start and stop of apache """
start = {'version': 1.0, 'method': 'startServer'}
stop = {'version': 1.0, 'method': 'stopServer'}
config = {
'version': 1.0,
'method': 'setConfig',
'args': {
'StartServers': 2
}
}
magi.util.execl.execDebug = True # Don't run, just log
self.fixture.inject(
MAGIMessage(src='guinode', data=yaml.safe_dump(config)))
self.fixture.inject(
MAGIMessage(src='guinode', data=yaml.safe_dump(start)))
self.fixture.inject(
MAGIMessage(src='guinode', data=yaml.safe_dump(stop)))
time.sleep(0.5)
self.assertEquals(magi.util.execl.execCalls[0], "apache2ctl -k start")
self.assertEquals(magi.util.execl.execCalls[1], "apache2ctl -k stop")
def unloadAll(self):
""" Unload all the process agents
by sending a stop message on all the agent transports """
log.info("Unloading all external agents")
for transport, dockList in self.invDockMap.iteritems():
call = {'version': 1.0, 'method': 'stop', 'args': {}}
stop_msg = MAGIMessage(docks=list(dockList)[0],
contenttype=MAGIMessage.YAML,
data=yaml.safe_dump(call))
request = AgentRequest.MAGIMessage(stop_msg)
log.debug('Sending stop message on transport %s, dock %s',
transport,
list(dockList)[0])
transport.outmessages.append(request)
#Waiting for agents to unload
for i in range(10):
if len(self.invDockMap) == 0:
log.debug("Agents unload done")
break
time.sleep(0.1) #waiting for the unload to be done
def transportAdded(self, transport):
""" When a transport comes up, add it to our list """
newtgl = TransportGroupList(self.msgintf, transport.fileno())
self.transportGroupLists[transport.fileno()] = newtgl
# Remember, rxGroups = Union(all other txgroups), need to rebuild this one as its blank right now
for othertgl in self.transportGroupLists.itervalues():
if othertgl is newtgl: continue
newtgl.rxGroups.incGroup(othertgl.txGroups.keys())
log.info("added transport %s, init rxgroup to: %s", transport, newtgl.rxGroups)
# ask neighbors to resend
resend = MAGIMessage(contenttype=MAGIMessage.YAML, groups=[GroupRouter.ONEHOPNODES], docks=[GroupRouter.DOCK], data=yaml.safe_dump({'resend':True}))
resend._routed = [transport.fileno()]
log.info("Sending transport add message %s", resend)
self.msgintf.send(resend)
#if len(newtgl.rxGroups) > 0: #Always send out a group list, as neighbors might have a stale copy
log.debug("Sending a group list out for the newly initialized transport")
response = { 'set': list(newtgl.rxGroups), 'count': len(newtgl.rxGroups), 'checksum': listChecksum(newtgl.rxGroups) }
listmsg = MAGIMessage(contenttype=MAGIMessage.YAML, groups=[GroupRouter.ONEHOPNODES], docks=[GroupRouter.DOCK], data=yaml.safe_dump(response))
listmsg._routed = [transport.fileno()]
self.msgintf.send(listmsg)
def trigger(self, **kwargs):
self.send(
MAGIMessage(groups="control",
docks="daemon",
data=yaml.dump(kwargs),
contenttype=MAGIMessage.YAML))
def test_MethodCall(self):
""" Test MethodCall code """
class tester(object):
def __init__(self, ut):
self.ut = ut
def kwargs(self, msg, **kwargs):
self.ut.assertEquals(kwargs['name1'], 'name1')
self.ut.assertEquals(kwargs['name2'], 'name2')
self.ut.assert_('name3' not in kwargs)
self.ut.assertEquals(kwargs['ignoreme'], 'extra')
def nameddefaults(self, msg, name1=None, name2=None, name3=None):
self.ut.assertEquals(name1, 'name1')
self.ut.assertEquals(name2, 'name2')
self.ut.assertEquals(name3, None)
def partdefaults(self, msg, name1, name2, name3=None):
self.ut.assertEquals(name1, 'name1')
self.ut.assertEquals(name2, 'name2')
self.ut.assertEquals(name3, None)
# decoding
self.assertRaises(CallException,
MethodCall,
request={
'version': 2.0,
'method': 'ping'
})
m = MethodCall(request={'version': 1.0, 'method': 'ping'})
# call types
call = {
'version': 1.0,
'method': 'kwargs',
'args': {
'name1': 'name1',
'name2': 'name2',
'ignoreme': 'extra'
}
}
testobj = tester(self)
testobj.name = 'testobj'
msg = yaml.safe_dump(call)
dispatchCall(testobj, msg, call)
call['method'] = 'nameddefaults'
msg = MAGIMessage(data=yaml.safe_dump(call))
dispatchCall(testobj, msg, call)
call['method'] = 'partdefaults'
msg = MAGIMessage(data=yaml.safe_dump(call))
dispatchCall(testobj, msg, call)
del call['args']['name2'] # missing required argument
msg = MAGIMessage(data=yaml.safe_dump(call))
logging.disable(40)
self.assertRaises(CallException, dispatchCall, testobj, msg, call)
logging.disable(0)
call['method'] = 'kwargs'
call['args']['name2'] = 'name2'
msg = MAGIMessage(data=yaml.safe_dump(call))
doMessageAction(testobj, msg)
call['trigger'] = 'MyAwesomeTrigger'
msg = MAGIMessage(data=yaml.safe_dump(call))
doMessageAction(testobj, msg, self.d.messaging)
from magi.util.agent import Agent
from magi.util.processAgent import initializeProcessAgent
#logging.basicConfig(level=0) #logging.DEBUG)
log = logging.getLogger("testagent")
if __name__ == '__main__':
name = sys.argv[1]
dock = sys.argv[2]
args = sys.argv[3:]
agent = Agent()
initializeProcessAgent(agent, sys.argv)
msg = agent.messenger.next()
log.debug("message in is %s" % msg)
if msg.data == 'testdata':
log.debug("data is testdata, sending response")
agent.messenger.send(MAGIMessage(nodes='somewhere',
docks='hello',
data="response from testAgentPipe"),
acknowledgement=True,
timestamp=98765)
else:
log.debug("data was %s, not testdata", msg.data)
log.debug("message sending done")
justwait = agent.messenger.next()
agent.done = True
def _sendTrigger(self, **data):
self.messaging.inject(
MAGIMessage(groups="control",
docks="control",
data=yaml.dump(data.copy())))