def prepare(self): """Prepare for this test - does test.doPrepare() - sends prepare messages to target probes - waits for all targets to reply with ready """ # prepare everyone self.test.doPrepare() for target in self.test.getTargets(): ProbeStorage.connectToProbe(target) # prepare for the test width given id Client.send( Prepare(target, self.test.getId(), self.test.getName(), self.test.getOptions(), Identification.PROBE_ID)) # wait for everyone to be ready self.isReadyForTest.wait(self.test.prepareTimeout) if not self.isReadyForTest.is_set(): # TODO: send abort ? self.testError = TestError( "Prepare action timed out, probes did not reply in time") self.isReadyForTest.clear() if self.testError: raise self.testError testLogger.info("Prepare over, executing test")
def __init__(self): self.helper = self.Helper(self) self.listener = Params.PROTOCOL.Listener(self.helper) #init the thread Thread.__init__(self) self.setName("Server") self.isUp = Event() ProbeStorage.addSelfProbe()
def manageRemove(cls, action): """Remove a probe from the currently known probes :param action: Remove action describing the probe to remove""" assert isinstance(action, a.Remove) cls.logger.debug("Managing Remove task") try: cls.logger.info("Removing %s from known probes", action.getIdSonde()) # remove probe from DHT ProbeStorage.delProbeById(action.getIdSonde()) except NoSuchProbe: cls.logger.warning("Probe not found in hash table")
def manageAdd(cls, action): """Add a probe to the DHT :param action: Add action containing the probe to add """ assert isinstance(action, a.Add) cls.logger.debug("Managing Add task") #add the probe to the local DHT ProbeStorage.addProbe(ProbeStorage.newProbe(action.getIdSonde(), action.getIpSonde())) if action.hello is not None: # tell the new probe about all other probe Client.send(action.hello) cls.logger.info("Added probe %s, id %s to known probes", action.getIpSonde(), action.getIdSonde())
def manageUpdateProbes(cls, action): """Update your list of probes with this set of probes :param action: UpdateProbes action instance """ assert isinstance(action, a.UpdateProbes) cls.logger.info("Joined overlay size %s", len(action.getProbeList())) if action.echo is not None: Client.send(Hello(action.echo, list(ProbeStorage.getAllOtherProbes()), Identification.PROBE_ID)) cls.logger.info("Sent echo to %s", action.echo) for probe in action.getProbeList(): # don't re-add ourselves to the local DHT if probe.getId() != Identification.PROBE_ID: ProbeStorage.addProbe(ProbeStorage.newProbe(probe.getId(), probe.getIp()))
def manageAdd(cls, action): """Add a probe to the DHT :param action: Add action containing the probe to add """ assert isinstance(action, a.Add) cls.logger.debug("Managing Add task") #add the probe to the local DHT ProbeStorage.addProbe( ProbeStorage.newProbe(action.getIdSonde(), action.getIpSonde())) if action.hello is not None: # tell the new probe about all other probe Client.send(action.hello) cls.logger.info("Added probe %s, id %s to known probes", action.getIpSonde(), action.getIdSonde())
def manageUpdateProbes(cls, action): """Update your list of probes with this set of probes :param action: UpdateProbes action instance """ assert isinstance(action, a.UpdateProbes) cls.logger.info("Joined overlay size %s", len(action.getProbeList())) if action.echo is not None: Client.send( Hello(action.echo, list(ProbeStorage.getAllOtherProbes()), Identification.PROBE_ID)) cls.logger.info("Sent echo to %s", action.echo) for probe in action.getProbeList(): # don't re-add ourselves to the local DHT if probe.getId() != Identification.PROBE_ID: ProbeStorage.addProbe( ProbeStorage.newProbe(probe.getId(), probe.getIp()))
def _initiateBroadcast(cls, message, toMyself): cls.logger.debug("Initiating broadcast message %s", message.__class__.__name__) # propagation phase prop = ProbeStorage.getIdAllOtherProbes() if toMyself: # make sure we are the first on our list prop.insert(0, Identification.PROBE_ID) # Only do something if there is something to do if len(prop) > 0: if len(prop) <= Consts.PROPAGATION_RATE: for p in prop: mes = copy.deepcopy(message) mes.targetId = p mes.recipientId = p cls.send(mes) else: pRate = Consts.PROPAGATION_RATE # take targets for first hop out of the list sendTo = prop[0:pRate] pt = prop[pRate:] propTargets = [pt[i::pRate] for i in range(pRate)] for i, firstHop in enumerate(sendTo): cls.send( BroadCast(firstHop, Identification.PROBE_ID, message, propTargets[i]))
def treatMessage(cls, message): """Handles the receptions of a Message (called by the listener) For regular Actions, addTask is called after translation of the message For TesterMessages and TesteeMessages, treatTestMessage is called :param message: The Message instance to treat """ cls.logger.debug("Treating message %s", message.__class__.__name__) assert isinstance(message, Message) # forwarding mechanism if message.targetId != Identification.PROBE_ID: if ProbeStorage.isKnownId(message.targetId): message.recipientId = message.targetId else: #if we already forwarded it before , stop here if message.hash in cls.forwardedMessages: cls.logger.warning( "Throwing message %s in forward because message was previously forwarded.", message.__class__.__name__) return Scheduler.forward() message.recipientId = ProbeStorage.getOtherRandomId() cls.logger.info("Forwarding message %s for %s to id %s", message.__class__.__name__, message.targetId, message.recipientId) cls.forwardedMessages.append(message.hash) Client.send(message) return # handle special class of messages separately if isinstance(message, TestMessage): cls.treatTestMessage(message) elif isinstance(message, WatcherMessage): cls.treatWatcherMessage(message) elif isinstance(message, BroadCast): # broadcast = do required action first and continue broadcast cls.logger.ddebug("Handling Broadcast") try: ActionMan.addTask(MTA.toAction(message.getMessage())) except ActionError: pass # be sure to propagate broadcast if a reasonable error occurs ActionMan.addTask(MTA.toAction(message)) # Client.broadcast(message) else: # handles everything else, including Do messages ActionMan.addTask(MTA.toAction(message))
def manageQuit(cls, action): """Quit the overlay nicely Tells everyone about this :param action: Quit action """ assert isinstance(action, a.Quit) cls.logger.debug("Managing Quit task") cls.logger.info("Exiting the overlay") try: Client.broadcast(Bye("", Identification.PROBE_ID), toMyself=False) except ProbeConnectionException as e: cls.logger.warning("Could not sent Bye message %s", e) # Other commands to close all connections, etc Client.allMessagesSent() ProbeStorage.clearAllProbes() cls.logger.info("All probes cleared, all connections closed.") ProbeStorage.addSelfProbe() cls.logger.info("Re-added the localhost probe, ready to proceed again")
def manageQuit(cls, action): """Quit the overlay nicely Tells everyone about this :param action: Quit action """ assert isinstance(action, a.Quit) cls.logger.debug("Managing Quit task") cls.logger.info("Exiting the overlay") try: Client.broadcast(Bye("", Identification.PROBE_ID), toMyself = False) except ProbeConnectionException as e: cls.logger.warning("Could not sent Bye message %s", e) # Other commands to close all connections, etc Client.allMessagesSent() ProbeStorage.clearAllProbes() cls.logger.info("All probes cleared, all connections closed.") ProbeStorage.addSelfProbe() cls.logger.info("Re-added the localhost probe, ready to proceed again")
def treatMessage(cls, message): """Handles the receptions of a Message (called by the listener) For regular Actions, addTask is called after translation of the message For TesterMessages and TesteeMessages, treatTestMessage is called :param message: The Message instance to treat """ cls.logger.debug("Treating message %s", message.__class__.__name__) assert isinstance(message, Message) # forwarding mechanism if message.targetId != Identification.PROBE_ID: if ProbeStorage.isKnownId(message.targetId): message.recipientId = message.targetId else: #if we already forwarded it before , stop here if message.hash in cls.forwardedMessages: cls.logger.warning("Throwing message %s in forward because message was previously forwarded.", message.__class__.__name__) return Scheduler.forward() message.recipientId = ProbeStorage.getOtherRandomId() cls.logger.info("Forwarding message %s for %s to id %s", message.__class__.__name__, message.targetId, message.recipientId) cls.forwardedMessages.append(message.hash) Client.send(message) return # handle special class of messages separately if isinstance(message, TestMessage): cls.treatTestMessage(message) elif isinstance(message, WatcherMessage): cls.treatWatcherMessage(message) elif isinstance(message, BroadCast): # broadcast = do required action first and continue broadcast cls.logger.ddebug("Handling Broadcast") try: ActionMan.addTask(MTA.toAction(message.getMessage())) except ActionError: pass # be sure to propagate broadcast if a reasonable error occurs ActionMan.addTask(MTA.toAction(message)) # Client.broadcast(message) else: # handles everything else, including Do messages ActionMan.addTask(MTA.toAction(message))
def sendMessage(self, message): """Send this message using the Params.PROTOCOL :param message: The message to send """ if not ProbeStorage.isKnownId(message.recipientId): self.logger.warning("The probe %s is not currently known to me, message will not be sent", message.targetId) return self.logger.debug("Sending the message : %s for %s to %s with ip %s", message.__class__.__name__, message.getTarget(), message.recipientId, ProbeStorage.getProbeById(message.recipientId).getIp()) try: Retry.retry(times = Consts.SEND_RETRY, interval = Consts.SEND_RETRY_INTERVAL, failure = ProbeConnectionException, eraise = ProbeConnectionException )(self.sender.send)(message) except ProbeConnectionException as e: raise SendError(e)
def sendMessage(self, message): """Send this message using the Params.PROTOCOL :param message: The message to send """ if not ProbeStorage.isKnownId(message.recipientId): self.logger.warning( "The probe %s is not currently known to me, message will not be sent", message.targetId) return self.logger.debug( "Sending the message : %s for %s to %s with ip %s", message.__class__.__name__, message.getTarget(), message.recipientId, ProbeStorage.getProbeById(message.recipientId).getIp()) try: Retry.retry(times=Consts.SEND_RETRY, interval=Consts.SEND_RETRY_INTERVAL, failure=ProbeConnectionException, eraise=ProbeConnectionException)( self.sender.send)(message) except ProbeConnectionException as e: raise SendError(e)
def prepare(self): """Prepare for this test - does test.doPrepare() - sends prepare messages to target probes - waits for all targets to reply with ready """ # prepare everyone self.test.doPrepare() for target in self.test.getTargets(): ProbeStorage.connectToProbe(target) # prepare for the test width given id Client.send(Prepare(target, self.test.getId(), self.test.getName(), self.test.getOptions(), Identification.PROBE_ID)) # wait for everyone to be ready self.isReadyForTest.wait(self.test.prepareTimeout) if not self.isReadyForTest.is_set(): # TODO: send abort ? self.testError = TestError("Prepare action timed out, probes did not reply in time") self.isReadyForTest.clear() if self.testError: raise self.testError testLogger.info("Prepare over, executing test")
def handleProbeQuery(cls): """Handle request to give the probes""" probes = ProbeStorage.getAllProbes() dprobes = [] for probe in probes: status = [] if probe.getId() == Identification.PROBE_ID: status.append(pd.ProbeStatus.LOCAL) status.append(pd.ProbeStatus.ADDED) if probe.connected: status.append(pd.ProbeStatus.CONNECTED) dprobes.append(pd.Probe(probe.getId(), probe.getIp(), pd.statusFactory(status))) return cconsts.Params.CODEC.encode(dprobes)
def handleProbeQuery(cls): """Handle request to give the probes""" probes = ProbeStorage.getAllProbes() dprobes = [] for probe in probes: status = [] if probe.getId() == Identification.PROBE_ID: status.append(pd.ProbeStatus.LOCAL) status.append(pd.ProbeStatus.ADDED) if probe.connected: status.append(pd.ProbeStatus.CONNECTED) dprobes.append( pd.Probe(probe.getId(), probe.getIp(), pd.statusFactory(status))) return cconsts.Params.CODEC.encode(dprobes)
def manageAddPrefix(cls, action): """Add a prefix to the DHT. A prefix is a set of addresses :param action: AddPrefix action """ assert isinstance(action, a.AddPrefix) try: net = ip_network(action.getPrefix(), strict = False) hosts = net.hosts() if net.num_addresses > 1 else [net.network_address] for host in hosts: try: h = str(host) if not ProbeStorage.isKnownIp(h): Client.send(AddToOverlay(Identification.PROBE_ID, h)) except Exception as e: cls.logger.warning("Error while adding probe %s : %s", h, e) except ValueError: cls.logger.warning("Wrong prefix given %s", action.getPrefix())
def send(self, message): """Send this message on the network :param message: Message instance to send """ try: target = ProbeStorage.getProbeById(message.recipientId) # serialize our message serializedMessage = Params.CODEC.encode(message) # put it in a dictionary params = {Parameters.POST_MESSAGE_KEYWORD: serializedMessage} # transform dictionary into string params = urllib.parse.urlencode( params, doseq=True, encoding=Parameters.POST_MESSAGE_ENCODING) # set the header as header for POST headers = { "Content-type": "application/x-www-form-urlencoded;charset=%s" % Parameters.POST_MESSAGE_ENCODING, "Accept": "text/plain" } urlQuery = "" if isinstance(message, TestMessage): urlQuery = Parameters.URL_SRV_TESTS_QUERY response = self._sendMessage(target, Parameters.HTTP_POST_REQUEST, urlQuery, params, headers) if response.status != 200: self.logger.warning("Wrong status received!") # self.send(message) except NoSuchProbe: self.logger.error( "The probe you requested to send a message to : '%s', is currently unknown to me.", message.recipientId) except socket.timeout as e: self.logger.warning( 'Timeout occurred while sending message to %s@%s', message.recipientId, target.getAddress()) raise ProbeConnectionException(e) except HTTPException as e: self.logger.error("Cannot send message to %s@%s", message.recipientId, target.getAddress()) self.logger.debug("Cannot send message", exc_info=1) raise ProbeConnectionException(e)
def manageAddPrefix(cls, action): """Add a prefix to the DHT. A prefix is a set of addresses :param action: AddPrefix action """ assert isinstance(action, a.AddPrefix) try: net = ip_network(action.getPrefix(), strict=False) hosts = net.hosts() if net.num_addresses > 1 else [ net.network_address ] for host in hosts: try: h = str(host) if not ProbeStorage.isKnownIp(h): Client.send(AddToOverlay(Identification.PROBE_ID, h)) except Exception as e: cls.logger.warning("Error while adding probe %s : %s", h, e) except ValueError: cls.logger.warning("Wrong prefix given %s", action.getPrefix())
def manageAddToOverlay(cls, action): assert isinstance(action, a.AddToOverlay) cls.logger.ddebug("Add probe to overlay") try: probeId = Retry.retry(times=Consts.GET_REMOTE_ID_RETRY, interval=Consts.GET_REMOVE_ID_RETRY_INTERVAL, failure=ProbeConnectionException, eraise=ProbeConnectionException)( Client.getRemoteId)(action.probeIp) # probeId = Client.getRemoteId(action.probeIp) cls.logger.info("Adding probe %s at %s to overlay", probeId, action.probeIp) addMessage = Add(Identification.PROBE_ID, probeId, action.probeIp) #use action directly because of timing issues selfAddAction = a.Add( action.probeIp, probeId, Hello(probeId, list(ProbeStorage.getAllOtherProbes()), Identification.PROBE_ID, echo=Identification.PROBE_ID if action.mergeOverlays else None)) # selfAddMessage = copy.deepcopy(addMessage) # selfAddAction.hello = Hello(probeId, # list(ProbeStorage.getAllOtherProbes()), # Identification.PROBE_ID, # echo = Identification.PROBE_ID if action.mergeOverlays else None) # Do broadcast before adding the probe so that it doesn't receive unnecessary message # addMessage = m.Add(Identification.PROBE_ID, probeId, message.targetIp, hello=True) # print(ProbeStorage.getIdAllOtherProbes()) Client.broadcast(addMessage) #treat message after so that the new guy does not receive bogus add message #treat the add for this addToOverlay before any other AddToOverlay # import calls.messagetoaction as MTA cls.addTask(selfAddAction) #try to fix adding host too quickly Scheduler.addToOverlay() cls.logger.debug("Probe %s added to overlay", probeId) except ProbeConnectionException as e: cls.logger.warning("Adding probe failed %s : %s", action.probeIp, e)
def _propagateBroadcast(cls, broadcast): assert isinstance(broadcast, BroadCast) prop = broadcast.getNextTargets() payload = broadcast.getMessage() cls.logger.debug("Propagating message %s to %s", broadcast.__class__.__name__, repr(prop)) # Only do something if there is something to do if len(prop) > 0: if len(prop) <= Consts.PROPAGATION_RATE: #in the end, send the actual message for p in prop: mes = copy.deepcopy(payload) mes.targetId = p mes.recipientId = p #if we know the target, send the message if ProbeStorage.isKnownId(p): cls.send(mes) elif ProbeStorage.isKnownId(broadcast.sourceId): #try to avoid breaking the chain during broadcasts #send back the payload to the initial source of the broadcast if we don't know this recipient # (forwarding at initial host will work) mes.recipientId = broadcast.sourceId cls.send(mes) else: mes.recipientId = ProbeStorage.getOtherRandomId() cls.send(mes) else: pRate = Consts.PROPAGATION_RATE # take targets for first hop out of the list sendTo = prop[0:pRate] pt = prop[pRate:] propTargets = [pt[i::pRate] for i in range(pRate)] for i, firstHop in enumerate(sendTo): nextHops = propTargets[i] #copy the sourceId of the original message when chaining m = BroadCast(firstHop, broadcast.sourceId, payload, nextHops) if ProbeStorage.isKnownId(firstHop): cls.send(m) elif ProbeStorage.isKnownId(broadcast.sourceId): m.recipientId = broadcast.sourceId cls.send(m) else: m.recipientId = ProbeStorage.getOtherRandomId() cls.send(m)
def _initiateBroadcast(cls, message, toMyself): cls.logger.debug("Initiating broadcast message %s", message.__class__.__name__) # propagation phase prop = ProbeStorage.getIdAllOtherProbes() if toMyself: # make sure we are the first on our list prop.insert(0, Identification.PROBE_ID) # Only do something if there is something to do if len(prop) > 0: if len(prop) <= Consts.PROPAGATION_RATE: for p in prop: mes = copy.deepcopy(message) mes.targetId = p mes.recipientId = p cls.send(mes) else: pRate = Consts.PROPAGATION_RATE # take targets for first hop out of the list sendTo = prop[0:pRate] pt = prop[pRate:] propTargets = [pt[i::pRate] for i in range(pRate)] for i, firstHop in enumerate(sendTo): cls.send(BroadCast(firstHop, Identification.PROBE_ID, message, propTargets[i]))
def manageAddToOverlay(cls, action): assert isinstance(action, a.AddToOverlay) cls.logger.ddebug("Add probe to overlay") try: probeId = Retry.retry(times = Consts.GET_REMOTE_ID_RETRY, interval = Consts.GET_REMOVE_ID_RETRY_INTERVAL, failure = ProbeConnectionException, eraise = ProbeConnectionException)(Client.getRemoteId)(action.probeIp) # probeId = Client.getRemoteId(action.probeIp) cls.logger.info("Adding probe %s at %s to overlay", probeId, action.probeIp) addMessage = Add(Identification.PROBE_ID, probeId, action.probeIp) #use action directly because of timing issues selfAddAction = a.Add(action.probeIp, probeId, Hello(probeId, list(ProbeStorage.getAllOtherProbes()), Identification.PROBE_ID, echo = Identification.PROBE_ID if action.mergeOverlays else None) ) # selfAddMessage = copy.deepcopy(addMessage) # selfAddAction.hello = Hello(probeId, # list(ProbeStorage.getAllOtherProbes()), # Identification.PROBE_ID, # echo = Identification.PROBE_ID if action.mergeOverlays else None) # Do broadcast before adding the probe so that it doesn't receive unnecessary message # addMessage = m.Add(Identification.PROBE_ID, probeId, message.targetIp, hello=True) # print(ProbeStorage.getIdAllOtherProbes()) Client.broadcast(addMessage) #treat message after so that the new guy does not receive bogus add message #treat the add for this addToOverlay before any other AddToOverlay # import calls.messagetoaction as MTA cls.addTask(selfAddAction) #try to fix adding host too quickly Scheduler.addToOverlay() cls.logger.debug("Probe %s added to overlay", probeId) except ProbeConnectionException as e: cls.logger.warning("Adding probe failed %s : %s", action.probeIp, e)
def quit(cls): """Terminate this instance properly""" cls.logger.info("Stopping the Client") cls._terminate() ProbeStorage.closeAllConnections()
def getProbeIpById(probeId): """Returns the Ip of a probe given it's Id :param probeId: ID of the probe""" return ProbeStorage.getProbeById(probeId).getIp()
def getIdAllOtherProbes(): """Returns the Ids of all the other known probes""" return ProbeStorage.getIdAllOtherProbes()
def getIpAllOtherProbes(cls): return ProbeStorage.getIpAllOtherProbes()
def result(self): """Disconnect probes and compute result of the test""" for target in self.test.getTargets(): ProbeStorage.disconnectFromProbe(target) self.test.doResult(self.reports) testLogger.info("Results processing over, test is done")
def getAllProbes(cls): return ProbeStorage.getAllProbes()