def test_RouteBack(self): """ Test for no routing out same interface if routeresponse comes from there """ # from n1 to n2,g1,g2 , send node request, get response on interface it came in out, don't send back that way, just drop msg = self.newMsg() msg.dstnodes = ['n2'] msg._receivedon = self.transports[10] rtmsg = MAGIMessage(src="n2", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response': True})) rtmsg._receivedon = self.transports[10] logging.disable(30) # disable warning for no route, expected self.push(msg) logging.disable(0) self.assertEquals( 1, len(self.router.processors[6].pausedMessages['n2'] )) # BIG DEPENDENCY ON PROCESSOR ORDERING in this test self.push(rtmsg) self.assertEquals(0, len(self.router.processors[6].pausedMessages['n2'])) self._checkForMessageOnlyIn("data", yaml.safe_dump({'request': 'n2'}), [10, 11, 12]) self._checkForMessageOnlyIn("msgid", 1234, []) self.assert_(self.router.rxqueue.empty(), "Local receive queue should be empty" ) # routerdock message absorbed by router processor self.commonAssert()
def _incomingGroupResend(self, fd, src): request = {'resend': True} msg = MAGIMessage(src=src, contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) msg._receivedon = self.transports[fd] self.router.processIN([msg], time.time())
def test_RequestResponse(self): """ Test basic response to a request """ rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'request':'mynode'})) rtmsg._receivedon = self.transports[1] self.router.processIN([rtmsg], time.time()) self.assertEquals(1, len(self.store.outgoing)) self.assertEquals(set([NodeRouter.DOCK]), self.store.outgoing[0].dstdocks) self.assert_('response' in yaml.load(self.store.outgoing[0].data))
def test_localGroup(self): """ Test message with a local group """ msg = MAGIMessage(groups=["mygroup"]) msg._receivedon = self.transports[1] # Local joins mygroup, message enters on transport1 for mygroup self.router.groupRequest(GroupRequest("join", "mygroup", "default")) fds = self.router.routeMessage(msg) self.assertEqual(fds, set([0]))
def test_localGroup(self): """ Test message with a local group """ msg = MAGIMessage(groups=["mygroup"]) msg._receivedon = self.transports[1] # Local joins mygroup, message enters on transport1 for mygroup self.router.groupRequest(GroupRequest("join", "mygroup", "default")) fds = self.router.routeMessage(msg) self.assertEqual(fds, set([0]))
def test_multipleRoutes(self): """ Test for multiple nodes including localhost as one """ msg = MAGIMessage(nodes=["unknown","mynode"]) msg._receivedon = self.transports[2] rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg._receivedon = self.transports[1] self.router.processIN([rtmsg], time.time()) fds = self.router.routeMessage(msg) self.assertEqual(fds, set([0,1]))
def test_RequestResponse(self): """ Test basic response to a request """ rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'request': 'mynode'})) rtmsg._receivedon = self.transports[1] self.router.processIN([rtmsg], time.time()) self.assertEquals(1, len(self.store.outgoing)) self.assertEquals(set([NodeRouter.DOCK]), self.store.outgoing[0].dstdocks) self.assert_('response' in yaml.load(self.store.outgoing[0].data))
def _incomingGroupList(self, fd, src, grouplist): self.neighbors[fd][src] = set(grouplist) request = { 'set': grouplist, 'count': len(self.neighbors[fd][src]), 'checksum': listChecksum(self.neighbors[fd][src]) } msg = MAGIMessage(src=src, contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) msg._receivedon = self.transports[fd] self.router.processIN([msg], time.time())
def test_routeNoBackwards(self): """ Test route and request without sending back out initial receiving interface """ msg = MAGIMessage(nodes=["unknown"]) msg._receivedon = self.transports[1] rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg._receivedon = self.transports[1] # Attempt to route message, should have nowhere to go fds = self.router.routeMessage(msg) self.assertEqual(fds, set([])) self._checkForMessageOnlyIn('data', yaml.safe_dump({'request':'unknown'}), [1, 2]) # Get a route response and check for proper queing self.router.processIN([rtmsg], time.time()) self._checkForMessageOnlyIn('dstnodes', ['unknown'], [])
def test_requestSquelch(self): """ Test to make sure squelching of requests occurs """ msg1 = MAGIMessage(nodes=["unknown"], data='1') msg1._receivedon = self.transports[1] msg2 = MAGIMessage(nodes=["unknown"], data='2') msg2._receivedon = self.transports[1] # Two messages for unknown, only one request should be queued fds = self.router.routeMessage(msg1) self.assertEqual(fds, set([])) fds = self.router.routeMessage(msg2) self.assertEqual(fds, set([])) self._checkForMessageOnlyIn('data', yaml.safe_dump({'request':'unknown'}), [1, 2]) self._checkMessagesInEquals(1, 1) self._checkMessagesInEquals(2, 1)
def getData(self, msg, agents=None, nodes=None, filters=dict(), timestampChunks=None, visited=set()): """ Request to fetch data """ functionName = self.getData.__name__ helpers.entrylog(log, functionName, locals()) agents_ = helpers.toSet(agents) nodes_ = helpers.toSet(nodes) if not nodes_: nodes_ = config.getTopoGraph().nodes() if not agents_: if nodes: agents_ = self.getSensorAgents(nodes[0]) else: raise AttributeError( "Cannot query for an empty set of collections.") if timestampChunks == None: timestampChunks = [(0, time.time())] data = dict() for agent in agents_: data[agent] = dict() for node in nodes_: filters_copy = filters.copy() filters_copy['host'] = node nodedata = [] for tsChunk in timestampChunks: nodedata = nodedata + database.getData( agent, filters_copy, tsChunk, database.configHost(), database.ROUTER_SERVER_PORT) data[agent][node] = nodedata args = { "agents": agents, "nodes": nodes, "filters": filters, "timestampChunks": timestampChunks, "visited": visited, "data": data } call = {'version': 1.0, 'method': 'putData', 'args': args} log.debug('Creating data message') msg = MAGIMessage(nodes=msg.src, docks='dataman', contenttype=MAGIMessage.PICKLE, data=pickle.dumps(call)) log.debug('Sending message') self.messenger.send(msg) helpers.exitlog(log, functionName)
def sendMsg(self): self.msgid += 1 msg = MAGIMessage() msg.msgid = self.msgid msg.contenttype = MAGIMessage.NONE msg.src = "servernode" msg.srcdock = "serverdock" msg.dstgroups = ['multicastgroup'] msg.data = "success" msg._routed = [self.conn.fileno()] self.messenger.thread.sendDirect(msg) while self.messenger.thread.pollMap[self.conn.fileno()].outmessages: time.sleep(0.2) #waiting for message to be sent
def sendRho(self, k): functionName = self.sendRho.__name__ helpers.entrylog(log, functionName, locals()) kwargs = {'method' : 'receiveRho', 'args' : {'k' : k, 'rho' : self.rho[k]}, 'version' : 1.0} log.debug("Sending rho: %s", kwargs['args']) msg = MAGIMessage(nodes="iso", docks="dmm_dock", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML) self.messenger.send(msg) helpers.exitlog(log, functionName)
def requestPdr(self, k): functionName = self.requestPdr.__name__ helpers.entrylog(log, functionName) log.debug("Requesting Pdr, k:%d", k) kwargs = {'method' : 'sendPdr', 'args' : {'k' : k}, 'version' : 1.0} msg = MAGIMessage(groups="dr_group", docks="dmm_dock", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML) self.messenger.send(msg) helpers.exitlog(log, functionName)
def sendFreqErrorNotice(self, status): functionName = self.sendFreqErrorNotice.__name__ helpers.entrylog(log, functionName, locals()) kwargs = {'method' : 'receiveFreqErrorNotice', 'args' : {'status' : status}, 'version' : 1.0} log.info("Sending freqErrorNotice: %s", kwargs['args']) msg = MAGIMessage(nodes="iso", docks="dmm_dock", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML) self.messenger.send(msg) helpers.exitlog(log, functionName)
def test_remoteGroup(self): """ Test message with a remote group and transport down """ addother = { 'add': ['othergroup'], 'count': 1, 'checksum': listChecksum(['othergroup']) } rtmsg1 = MAGIMessage(groups=[GroupRouter.ONEHOPNODES], contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(addother)) rtmsg1._receivedon = self.transports[2] adddead = { 'add': ['deadgroup'], 'count': 2, 'checksum': listChecksum(['deadgroup', 'othergroup']) } rtmsg2 = MAGIMessage(groups=[GroupRouter.ONEHOPNODES], contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(adddead)) rtmsg2._receivedon = self.transports[2] msg = MAGIMessage(groups=["othergroup"]) msg._receivedon = self.transports[1] # Local joins mygroup, on transport2 they join othergroup and deadgroup, message enters on transport1 for othergroup self.router.groupRequest(GroupRequest("join", "mygroup", "default")) self.router.processIN([rtmsg1], time.time()) self.router.processIN([rtmsg2], time.time()) fds = self.router.routeMessage(msg) self.assertEqual(fds, set([2])) # Now transport is flagged as down, same message should route to nowhere self.router.transportRemoved(2, self.transports[2]) msg._appendedto = set() fds = self.router.routeMessage(msg) self.assertEqual(fds, set())
def trigger(self, **kwargs): """ Send a trigger event. Single location for a common action, ick though, this is application level stuff in messaging code """ self.send( MAGIMessage(groups="control", docks="daemon", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML))
def test_routeUnknown(self): """ Test route without a destination node """ msg = MAGIMessage(src="thesrc", nodes=["unknown"]) msg.msgid = 12345 msg._receivedon = self.transports[1] rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg._receivedon = self.transports[2] # Attempt to route message, should have nowhere to go fds = self.router.routeMessage(msg) self.assertEqual(fds, set([])) self._checkForMessageOnlyIn('data', yaml.safe_dump({'request':'unknown'}), [1, 2]) # Get a route response and check for proper queing, original header information should stay intact when sending paused self.router.processIN([rtmsg], time.time()) self._checkForMessageOnlyIn('dstnodes', set(['unknown']), [2]) self._checkForMessageOnlyIn('src', "thesrc", [2]) self._checkForMessageOnlyIn('msgid', 12345, [2])
def sendMsg(self): self.msgid += 1 msg = MAGIMessage() msg.msgid = self.msgid msg.contenttype = MAGIMessage.NONE msg.src = "clientnode" msg.srcdock = "clientdock" msg.dstgroups = ['multicastgroup'] msg.data = "testing" msg._routed = [self.conn.fileno()] self.messenger.thread.sendDirect(msg) while self.messenger.thread.pollMap[self.conn.fileno()].outmessages: time.sleep(0.2) #waiting for message to be sent
def addMsg(self, msgid): self.msgid += 1 msg = MAGIMessage() msg.msgid = self.msgid msg.contenttype = MAGIMessage.NONE msg.src = "mynode" msg.srcdock = "sourcedock" msg.dstgroups = ['g1'] msg.data = None self.mct.processPacket( '192.168.1.1', 18808, MCTHeader.PktData(msgid, 1, 1).encode() + self.msgcodec.encode(msg))
def sendEdr(self, k): functionName = self.sendEdr.__name__ helpers.entrylog(log, functionName) for agentIndex in range(0, self.N_dem): node = "dr-"+str(agentIndex) edr = self.E_D[agentIndex, k] log.debug("Sending E_D, Node: %s, k:%d, E_D:%f", node, k, edr) kwargs = {'method' : 'receiveEdr', 'args' : {'k' : k, 'edr' : edr}, 'version' : 1.0} msg = MAGIMessage(nodes=node, docks="dmm_dock", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML) self.messenger.send(msg) helpers.exitlog(log, functionName)
def ping(self, msg): """ Alive like method call that will send a pong back to the caller """ args = {"server": config.getNodeName(), "result": "success"} call = {'version': 1.0, 'method': 'pong', 'args': args} msg = MAGIMessage(nodes=msg.src, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) self.messenger.send(msg)
def test_RouteBack(self): """ Test for no routing out same interface if routeresponse comes from there """ # from n1 to n2,g1,g2 , send node request, get response on interface it came in out, don't send back that way, just drop msg = self.newMsg() msg.dstnodes = ['n2'] msg._receivedon = self.transports[10] rtmsg = MAGIMessage(src="n2", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg._receivedon = self.transports[10] logging.disable(30) # disable warning for no route, expected self.push(msg) logging.disable(0) self.assertEquals(1, len(self.router.processors[6].pausedMessages['n2'])) # BIG DEPENDENCY ON PROCESSOR ORDERING in this test self.push(rtmsg) self.assertEquals(0, len(self.router.processors[6].pausedMessages['n2'])) self._checkForMessageOnlyIn("data", yaml.safe_dump({'request':'n2'}), [10, 11, 12]) self._checkForMessageOnlyIn("msgid", 1234, []) self.assert_(self.router.rxqueue.empty(), "Local receive queue should be empty") # routerdock message absorbed by router processor self.commonAssert()
def addMsg(self, msgid): self.msgid += 1 msg = MAGIMessage() msg.msgid = self.msgid msg.contenttype = MAGIMessage.NONE msg.src = "mynode" msg.srcdock = "sourcedock" msg.dstgroups = ['g1'] msg.data = None self.mct.processPacket('192.168.1.1', 18808, MCTHeader.PktData(msgid, 1, 1).encode() + self.msgcodec.encode(msg))
def test_BasicRoute(self): """ Test basic functionality of routeMessage in router core""" # local to n1, route request out all external, response on t10, route out t10, then response on t11, paused messages send out t11 rtmsg = MAGIMessage(src="n1", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response': True})) rtmsg._receivedon = self.transports[10] rtmsg2 = MAGIMessage(src="n2", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response': True})) rtmsg2._receivedon = self.transports[11] msg = self.newMsg() msg._receivedon = self.router.transportMap[0] self.push(rtmsg) self.push(msg) self._checkForMessageOnlyIn("data", yaml.safe_dump({'request': 'n1'}), []) # got response before msg, no requests self._checkForMessageOnlyIn("data", yaml.safe_dump({'request': 'n2'}), [10, 11, 12]) # want to know about n2 self._checkForMessageOnlyIn("msgid", 1234, [10]) self.assert_(self.router.rxqueue.empty(), "Local receive queue should be empty" ) # locally not n1,n2 or a member of g1 or g2 self.commonAssert() self.router.transportMap[10].outmessages = [] # clear self.push(rtmsg2) self._checkForMessageOnlyIn("msgid", 1234, [11])
def test_DoubleSend(self): """ Test for no routing out same interface twice for paused messages """ # from n1 to n2,g1, route requests on local, should deliver out t11 for g1, and then out t11 again for n1 but be squelched msg = self.newMsg() msg.dstnodes = ['n2'] msg._receivedon = self.router.transportMap[0] request = {'add': ['g1'], 'count': 1, 'checksum': listChecksum(['g1'])} rtmsg1 = MAGIMessage(src="ignore", groups=["__NEIGH__"], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) rtmsg1._receivedon = self.transports[11] rtmsg2 = MAGIMessage(src="n2", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response': True})) rtmsg2._receivedon = self.transports[11] self.push(rtmsg1) self.push(msg) self.push(rtmsg2) self._checkForMessageOnlyIn("data", yaml.safe_dump({'request': 'n2'}), [10, 11, 12]) self._checkForMessageOnlyIn("msgid", 1234, [11]) msgcount = len(self.transports[11].outmessages) self.assert_( msgcount == 2, "Should only be 1 request and 1 message in t11 queue, found %d" % msgcount) self.assert_(self.router.rxqueue.empty(), "Local receive queue should be empty") self.commonAssert()
def test_MultiRoute(self): """ Test multiple routes with loopback """ # from local to g1, no route requests, should deliver out t11 for g1, t12 for g2, not back to local as that was source request = {'add': ['g1'], 'count': 1, 'checksum': listChecksum(['g1'])} rtmsg1 = MAGIMessage(src="n1", groups=["__NEIGH__"], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) rtmsg1._receivedon = self.transports[11] request = {'add': ['g2'], 'count': 1, 'checksum': listChecksum(['g2'])} rtmsg2 = MAGIMessage(src="n2", groups=["__NEIGH__"], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) rtmsg2._receivedon = self.transports[12] msg = self.newMsg() msg.dstnodes = [] msg._receivedon = self.router.transportMap[0] self.router.routers[0].groupRequest( GroupRequest("join", "g1", "default")) self.push(rtmsg1) self.push(rtmsg2) self.push(msg) self._checkForMessageOnlyIn("dstdocks", ['RouteRequest'], []) self._checkForMessageOnlyIn("msgid", 1234, [11, 12]) self.assertEquals( self.router.rxqueue.qsize(), 1, "Local receive queue should have an incoming message") self.commonAssert()
def test_remoteGroup(self): """ Test message with a remote group and transport down """ addother = { 'add': ['othergroup'], 'count': 1, 'checksum': listChecksum(['othergroup']) } rtmsg1 = MAGIMessage(groups=[GroupRouter.ONEHOPNODES], contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(addother)) rtmsg1._receivedon = self.transports[2] adddead = { 'add': ['deadgroup'], 'count': 2, 'checksum': listChecksum(['deadgroup', 'othergroup']) } rtmsg2 = MAGIMessage(groups=[GroupRouter.ONEHOPNODES], contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(adddead)) rtmsg2._receivedon = self.transports[2] msg = MAGIMessage(groups=["othergroup"]) msg._receivedon = self.transports[1] # Local joins mygroup, on transport2 they join othergroup and deadgroup, message enters on transport1 for othergroup self.router.groupRequest(GroupRequest("join", "mygroup", "default")) self.router.processIN([rtmsg1], time.time()) self.router.processIN([rtmsg2], time.time()) fds = self.router.routeMessage(msg) self.assertEqual(fds, set([2])) # Now transport is flagged as down, same message should route to nowhere self.router.transportRemoved(2, self.transports[2]) msg._appendedto = set() fds = self.router.routeMessage(msg) self.assertEqual(fds, set())
def addBigMessage(self, mid, count, skip): msg = MAGIMessage() msg.msgid = 12345 msg.contenttype = MAGIMessage.NONE msg.src = "mynode" msg.srcdock = "sourcedock" msg.dstgroups = ['g1'] msg.data = "X" * (((TXMessageTracker.SPLITSIZE-4) * count) - len(self.msgcodec.encode(msg))) tracker = TXMessageTracker(msg=msg, multicastid=mid, codec=DefaultCodec()) for ii in range(1,count+1): if ii not in skip: self.mct.processPacket('192.168.1.1', 18808, tracker.getPart(ii))
def addBigMessage(self, mid, count, skip): msg = MAGIMessage() msg.msgid = 12345 msg.contenttype = MAGIMessage.NONE msg.src = "mynode" msg.srcdock = "sourcedock" msg.dstgroups = ['g1'] msg.data = "X" * (((TXMessageTracker.SPLITSIZE - 4) * count) - len(self.msgcodec.encode(msg))) tracker = TXMessageTracker(msg=msg, multicastid=mid, codec=DefaultCodec()) for ii in range(1, count + 1): if ii not in skip: self.mct.processPacket('192.168.1.1', 18808, tracker.getPart(ii))
def test_BasicRoute(self): """ Test basic functionality of routeMessage in router core""" # local to n1, route request out all external, response on t10, route out t10, then response on t11, paused messages send out t11 rtmsg = MAGIMessage(src="n1", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg._receivedon = self.transports[10] rtmsg2 = MAGIMessage(src="n2", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg2._receivedon = self.transports[11] msg = self.newMsg() msg._receivedon = self.router.transportMap[0] self.push(rtmsg) self.push(msg) self._checkForMessageOnlyIn("data", yaml.safe_dump({'request':'n1'}), []) # got response before msg, no requests self._checkForMessageOnlyIn("data", yaml.safe_dump({'request':'n2'}), [10,11,12]) # want to know about n2 self._checkForMessageOnlyIn("msgid", 1234, [10]) self.assert_(self.router.rxqueue.empty(), "Local receive queue should be empty") # locally not n1,n2 or a member of g1 or g2 self.commonAssert() self.router.transportMap[10].outmessages = [] # clear self.push(rtmsg2) self._checkForMessageOnlyIn("msgid", 1234, [11])
def test_MultiRoute(self): """ Test multiple routes with loopback """ # from local to g1, no route requests, should deliver out t11 for g1, t12 for g2, not back to local as that was source request = { 'add': ['g1'], 'count': 1, 'checksum': listChecksum(['g1']) } rtmsg1 = MAGIMessage(src="n1", groups=["__NEIGH__"], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) rtmsg1._receivedon = self.transports[11] request = { 'add': ['g2'], 'count': 1, 'checksum': listChecksum(['g2']) } rtmsg2 = MAGIMessage(src="n2", groups=["__NEIGH__"], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) rtmsg2._receivedon = self.transports[12] msg = self.newMsg() msg.dstnodes = [] msg._receivedon = self.router.transportMap[0] self.router.routers[0].groupRequest(GroupRequest("join", "g1", "default")) self.push(rtmsg1) self.push(rtmsg2) self.push(msg) self._checkForMessageOnlyIn("dstdocks", ['RouteRequest'], []) self._checkForMessageOnlyIn("msgid", 1234, [11, 12]) self.assertEquals(self.router.rxqueue.qsize(), 1, "Local receive queue should have an incoming message") self.commonAssert()
def test_DoubleSend(self): """ Test for no routing out same interface twice for paused messages """ # from n1 to n2,g1, route requests on local, should deliver out t11 for g1, and then out t11 again for n1 but be squelched msg = self.newMsg() msg.dstnodes = ['n2'] msg._receivedon = self.router.transportMap[0] request = { 'add': ['g1'], 'count': 1, 'checksum': listChecksum(['g1']) } rtmsg1 = MAGIMessage(src="ignore", groups=["__NEIGH__"], docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) rtmsg1._receivedon = self.transports[11] rtmsg2 = MAGIMessage(src="n2", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response':True})) rtmsg2._receivedon = self.transports[11] self.push(rtmsg1) self.push(msg) self.push(rtmsg2) self._checkForMessageOnlyIn("data", yaml.safe_dump({'request':'n2'}), [10, 11, 12]) self._checkForMessageOnlyIn("msgid", 1234, [11]) msgcount = len(self.transports[11].outmessages) self.assert_(msgcount == 2, "Should only be 1 request and 1 message in t11 queue, found %d" % msgcount) self.assert_(self.router.rxqueue.empty(), "Local receive queue should be empty") self.commonAssert()
def getSensorAgents(self, node): """ Internal function to fetch the list of sensor agents for a given node """ call = {'version': 1.0, 'method': 'getCollectionMetadata'} querymsg = MAGIMessage(nodes=node, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) queryHash = self.digest("CollectionMetadata", node) log.debug("getCollectionMetadata Query Hash: " + queryHash) self.events[queryHash] = threading.Event() self.messenger.send(querymsg) self.events[queryHash].wait() return self.collectionMetadata[node].value.keys()
def checkIfUp(self, host, timeout=5): """ Test call to check if data manager agent is available on a given node """ call = {'version': 1.0, 'method': 'ping'} msg = MAGIMessage(nodes=host, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) self.messenger.send(msg) stop = time.time() + timeout while time.time() < stop: if host in self.rcvdPongs: return True return False
def sendTheta(self, k): functionName = self.sendTheta.__name__ helpers.entrylog(log, functionName) theta = self.theta[:, k] kwargs = { 'method': 'receiveTheta', 'args': { 'k': k, 'theta': theta }, 'version': 1.0 } msg = MAGIMessage(nodes="grid", docks="dmm_dock", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML) self.messenger.send(msg) helpers.exitlog(log, functionName)
def sendMessage(bridgeNode, bridgePort, nodes, docks, method, args): # Join the overlay at the specified bridge node. messaging = api.ClientConnection(CLIENT_NAME, bridgeNode, bridgePort) # Create a ping message and send on the overlay # All node on the overlay will receive it and the daemon will respond with a pong message msg = MAGIMessage(nodes=nodes, docks=docks, contenttype=MAGIMessage.YAML, data=yaml.safe_dump({ 'method': method, 'args': args, 'version': 1.0 })) log.debug("Sending msg: %s" % (msg)) messaging.send(msg) return messaging
def test_multipleRoutes(self): """ Test for multiple nodes including localhost as one """ msg = MAGIMessage(nodes=["unknown", "mynode"]) msg._receivedon = self.transports[2] rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response': True})) rtmsg._receivedon = self.transports[1] self.router.processIN([rtmsg], time.time()) fds = self.router.routeMessage(msg) self.assertEqual(fds, set([0, 1]))
def getCollectionMetadata(self, msg): """ Request for collector information """ functionName = self.getCollectionMetadata.__name__ helpers.entrylog(log, functionName, locals()) args = {"collectionMetadata": database.collectionHosts} call = { 'version': 1.0, 'method': 'putCollectionMetadata', 'args': args } msg = MAGIMessage(nodes=msg.src, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) self.messenger.send(msg) helpers.exitlog(log, functionName)
def sendPg(self, msg, k): functionName = self.sendPg.__name__ helpers.entrylog(log, functionName) self.currentGridTs = k if k > self.lastPgRcvdTs: self.P_G[k] = self.P_G[self.lastPgRcvdTs] pg = self.P_G[k] log.info("Sending P_G to grid agent: %f (k=%d)", pg, k) kwargs = { 'method': 'receivePg', 'args': { 'k': k, 'pg': pg }, 'version': 1.0 } msg = MAGIMessage(nodes="grid", docks="dmm_dock", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML) self.messenger.send(msg) helpers.exitlog(log, functionName)
def test_requestSquelch(self): """ Test to make sure squelching of requests occurs """ msg1 = MAGIMessage(nodes=["unknown"], data='1') msg1._receivedon = self.transports[1] msg2 = MAGIMessage(nodes=["unknown"], data='2') msg2._receivedon = self.transports[1] # Two messages for unknown, only one request should be queued fds = self.router.routeMessage(msg1) self.assertEqual(fds, set([])) fds = self.router.routeMessage(msg2) self.assertEqual(fds, set([])) self._checkForMessageOnlyIn('data', yaml.safe_dump({'request': 'unknown'}), [1, 2]) self._checkMessagesInEquals(1, 1) self._checkMessagesInEquals(2, 1)
def pingCall(node, bridge, timeout=10): """ Test call to check if data manager agent is available on a given node """ messenger = getMessenger(bridge, 18808) call = {'version': 1.0, 'method': 'ping'} msg = MAGIMessage(nodes=node, docks='dataman', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) messenger.send(msg) start = time.time() stop = start + timeout current = start while current < stop: try: msg = messenger.nextMessage(True, timeout=0.2) if msg.src == node: data = yaml.load(msg.data) return data except Queue.Empty: pass current = time.time() raise IOError("No reply from %s" % (node))
def getAgentsProcessInfo(node, bridge='127.0.0.1', timeout=10): """ Function to request process information for active agents on a given node """ messenger = getMessenger(bridge, 18808) call = {'version': 1.0, 'method': 'getStatus', 'args': {'agentInfo': True}} msg = MAGIMessage(nodes=node, docks='daemon', contenttype=MAGIMessage.YAML, data=yaml.dump(call)) messenger.send(msg) start = time.time() stop = start + timeout current = start while current < stop: try: msg = messenger.nextMessage(True, timeout=0.2) if msg.src == node: data = yaml.load(msg.data) return data['agentInfo'] except Queue.Empty: pass current = time.time() raise IOError("No reply from %s" % (node))
def test_routeUnknown(self): """ Test route without a destination node """ msg = MAGIMessage(src="thesrc", nodes=["unknown"]) msg.msgid = 12345 msg._receivedon = self.transports[1] rtmsg = MAGIMessage(src="unknown", groups=["__ALL__"], docks=[NodeRouter.DOCK], data=yaml.safe_dump({'response': True})) rtmsg._receivedon = self.transports[2] # Attempt to route message, should have nowhere to go fds = self.router.routeMessage(msg) self.assertEqual(fds, set([])) self._checkForMessageOnlyIn('data', yaml.safe_dump({'request': 'unknown'}), [1, 2]) # Get a route response and check for proper queing, original header information should stay intact when sending paused self.router.processIN([rtmsg], time.time()) self._checkForMessageOnlyIn('dstnodes', set(['unknown']), [2]) self._checkForMessageOnlyIn('src', "thesrc", [2]) self._checkForMessageOnlyIn('msgid', 12345, [2])
def test_GroupAckAggregation(self): """ Test aggregation of group acknowledgments """ self.transports[3] = TestTransport(3) self.router.transportAdded(self.transports[3]) self.neighbors = {0: {}, 1: {}, 2: {}, 3: {}} for k in self.transports: self.neighbors[k] = defaultdict(set) self._incomingGroupAdd(1, 'n11', ['g1', 'g2']) self._incomingGroupAdd(1, 'n12', ['g3']) self._incomingGroupAdd(2, 'n21', ['g1', 'g2']) self._incomingGroupAdd(2, 'n22', ['g1', 'g3']) self._checkGroupState() # Check the processing of a message with groups that wants an ack, verify data structure inside msg = MAGIMessage(src="n30", nodes=["n21"], groups=['g1', 'g3']) msg.msgid = 789 msg.flags |= MAGIMessage.WANTACK msg._receivedon = self.transports[3] msg._routed = self.router.routeMessage(msg) self.assertEqual(msg._routed, set([1, 2])) out = self.router.processFWD([msg], time.time()) self.assertEquals(out, [msg]) key = ("n30", 789) self.assertEquals(self.router.ackHolds.keys(), [key]) self.assertEquals(sorted(self.router.ackHolds[key].keys()), ['g1', 'g3']) self.assertEquals(self.router.ackHolds[key]['g1'][1], set(['n11'])) self.assertEquals(self.router.ackHolds[key]['g1'][2], set(['n21', 'n22'])) self.assertEquals(self.router.ackHolds[key]['g3'][1], set(['n12'])) self.assertEquals(self.router.ackHolds[key]['g3'][2], set(['n22'])) self.store.outgoing = [] # clear # from N11, ack stops here ack = msg.createAck("", ['g1']) ack._receivedon = self.transports[1] ack.src = "n11" out = self.router.processFWD([ack], time.time()) # no group routing, only node routing self.assertEquals(self.router.ackHolds[key]['g1'].keys(), [2]) # nothing left on transport 1 for g1 self.assertEquals(0, len(out)) # from N21, ack passes with n21 but no groups ack = msg.createAck("n21", ['g1']) ack.src = "n21" ack._receivedon = self.transports[2] out = self.router.processFWD([ack], time.time()) self.assertEquals(self.router.ackHolds[key]['g1'][2], set(['n22'])) self.assertEquals(1, len(out)) self.assertEquals("n21", out[0].data) # from N22, ack passes with g1 ack = msg.createAck("", ['g1', 'g3']) ack.src = "n22" ack._receivedon = self.transports[2] out = self.router.processFWD([ack], time.time()) self.assertEquals(self.router.ackHolds[key].keys(), ['g3']) # g1 complete self.assertEquals(self.router.ackHolds[key]['g3'].keys(), [1]) # g3 is still active on transport 1 self.assertEquals(1, len(out)) self.assertEquals(",g1", out[0].data) # from N12, ack passes with g2 ack = msg.createAck("", ['g3']) ack.src = "n12" ack._receivedon = self.transports[1] out = self.router.processFWD([ack], time.time()) self.assertEquals(self.router.ackHolds.keys(), []) # all complete self.assertEquals(1, len(out)) self.assertEquals(",g3", out[0].data)
def test_routeNeighExt(self): """ Test internal message sent to NEIGH """ msg = MAGIMessage(groups=[GroupRouter.ONEHOPNODES]) msg._receivedon = self.transports[0] fds = self.router.routeMessage(msg) self.assertEqual(fds, set([1,2]))
def test_routeNone(self): """ Test message without any groups """ msg = MAGIMessage(nodes=["somenode"]) msg._receivedon = self.transports[1] fds = self.router.routeMessage(msg) self.assertEqual(fds, set([]))
def _incomingGroupList(self, fd, src, grouplist): self.neighbors[fd][src] = set(grouplist) request = { 'set': grouplist, 'count': len(self.neighbors[fd][src]), 'checksum': listChecksum(self.neighbors[fd][src]) } msg = MAGIMessage(src=src, contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) msg._receivedon = self.transports[fd] self.router.processIN([msg], time.time())
def _incomingGroupResend(self, fd, src): request = { 'resend': True } msg = MAGIMessage(src=src, contenttype=MAGIMessage.YAML, docks=[GroupRouter.DOCK], data=yaml.safe_dump(request)) msg._receivedon = self.transports[fd] self.router.processIN([msg], time.time())
def test_encodeAndDecode(self): """ Test encoding and decoding of a message header """ msg = MAGIMessage() msg.msgid = 1234 msg.flags = 0x63 msg.contenttype = MAGIMessage.YAML msg.src = "mynode" msg.srcdock = "sourcedock" msg.hmac = "123456789" msg.dstnodes = set(['n1', 'n2']) msg.dstgroups = set(['g1', 'g2']) msg.dstdocks = set(['d1', 'd2']) msg.sequence = 98765 msg.timestamp = 12347890 msg.data = None codec = DefaultCodec() hdr = codec.encode(msg) ret, hdrsize = codec.decode(hdr) ret.data = None self.assertEquals(hdrsize, len(hdr)) for k, v in msg.__dict__.iteritems(): if k[0] == '_': continue self.assertEquals(getattr(ret, k), v)
def test_routeAll(self): """ Test route with a group ALL """ msg = MAGIMessage(groups=[GroupRouter.ALLNODES]) msg._receivedon = self.transports[1] fds = self.router.routeMessage(msg) self.assertEqual(fds, set([0,1,2]))
def newMsg(self): msg = MAGIMessage() msg.msgid = 1234 msg.flags = 0x0 msg.contenttype = MAGIMessage.YAML msg.src = "mynode" msg.srcdock = "sourcedock" msg.hmac = "123456789" msg.dstnodes = ['n1', 'n2'] msg.dstgroups = ['g1', 'g2'] msg.dstdocks = ['d1', 'd2'] msg.sequence = 98765 msg.timestamp = 12347890 msg.data = "helloworld" return msg