def main(args): CONFIG.register_descriptions({ "speedtest.client.uri": "Base URI to connect to", "speedtest.client.nconn": "Number of concurrent connections to use", "speedtest.client.latency_tries": "Number of latency measurements", }) common.main("speedtest.client", "Speedtest client", args) conf = CONFIG.copy() # # If possible use the runner, which will execute the # test in the context of the neubot daemon. Then exit # to bypass the POLLER.loop() invokation that is below # here. # If the runner fails, fallback to the usual code path, # which executes the test in the context of the local # process. # Set 'runned.enabled' to 0 to bypass the runner and # run the test locally. # if (utils.intify(conf['runner.enabled']) and runner_clnt.runner_client(conf["agent.api.address"], conf["agent.api.port"], LOG.noisy, "speedtest")): sys.exit(0) LOG.info('Will run the test in the local context...') client = ClientSpeedtest(POLLER) client.configure(conf) client.connect_uri() POLLER.loop()
def connect_uri(self, uri=None, count=None): if not uri: uri = self.conf.get("speedtest.client.uri", "http://master.neubot.org/") if not count: count = self.conf.get("speedtest.client.nconn", 1) LOG.info("* speedtest with %s" % uri) ClientHTTP.connect_uri(self, uri, count)
def run(poller, conf): """ Load MaxMind database and register our child server """ GEOLOCATOR.open_or_die() LOG.info("This product includes GeoLite data created by MaxMind, " "available from <http://www.maxmind.com/>.") server = ServerRendezvous(None) server.configure(conf) HTTP_SERVER.register_child(server, "/rendezvous")
def run(self, test, callback, auto_rendezvous=True, ctx=None): ''' Run test and callback() when done ''' # # If we are about to run a test and the list of # available tests is empty, we need certainly to # refill it before proceeding. # if (auto_rendezvous and test != 'rendezvous' and len(RUNNER_TESTS.get_test_names()) == 0): LOG.info('runner_core: Need to rendezvous first...') self.queue.append(('rendezvous', lambda: None, None)) self.queue.append((test, callback, ctx)) self.run_queue()
def open_or_die(self): path = CONFIG.get("rendezvous.geoip_wrapper.country_database", COUNTRY_DATABASE) # # Detect the common error case, i.e. that the user has # not downloaded the database. If something fancy is # going on, let the GeoIP library stacktrace for us. # if not os.path.exists(path): LOG.error("Missing GeoLiteCountry database: %s" % path) LOG.info("Please download it from " "<http://www.maxmind.com/app/geolitecountry>.") sys.exit(1) self.countries = GeoIP.open(path, GeoIP.GEOIP_STANDARD)
def _schedule(self): # # Typically the user does not specify the interval # and we use a random value around 1500 seconds. # The random value is extracted just once and from # that point on we keep using it. # interval = CONFIG["agent.interval"] if not interval: if not self._interval: self._interval = 1380 + random.randrange(0, 240) interval = self._interval LOG.info("* Next rendezvous in %d seconds" % interval) fn = lambda *args, **kwargs: self.connect_uri() self._task = POLLER.sched(interval, fn) STATE.update("idle", publish=False) STATE.update("next_rendezvous", self._task.timestamp)
def got_response(self, stream, request, response): if response.code != "200": LOG.complete("bad response") self._schedule() else: LOG.complete() s = response.body.read() try: m1 = marshal.unmarshal_object(s, "application/json", compat.RendezvousResponse) except ValueError: LOG.exception() self._schedule() else: if "version" in m1.update and "uri" in m1.update: ver, uri = m1.update["version"], m1.update["uri"] LOG.info("Version %s available at %s" % (ver, uri)) STATE.update("update", {"version": ver, "uri": uri}) _open_browser_on_windows("update.html") # Update tests known by the runner runner_lst.update(m1.available) # # Choose the test we would like to run even if # we're not going to run it because we're running # in debug mode or tests are disabled. # This allows us to print to the logger the test # we /would/ have choosen if we were allowed to run # it. # test = runner_lst.get_next_test() if not test: LOG.warning("No test available") self._schedule() return LOG.info("* Chosen test: %s" % test) # Are we allowed to run a test? if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]: LOG.info("Tests are disabled... not running") self._schedule() else: # Do we have negotiate URI for test? negotiate_uri = runner_lst.test_to_negotiate_uri(test) if not negotiate_uri: LOG.warning("No negotiate URI for test") self._schedule() else: # Actually run the test runner_core.run(test, negotiate_uri, self._schedule)
def process_request(self, stream, request): m = marshal.unmarshal_object(request.body.read(), "application/xml", compat.RendezvousRequest) m1 = compat.RendezvousResponse() version = self.conf["rendezvous.server.update_version"] # # Don't offer a release candidate update if the user is not # running a release candidate as well and viceversa. # if (("-rc" in version and "-rc" in m.version) or (not "-rc" in version and not "-rc" in m.version)): if m.version and LibVersion.compare(version, m.version) > 0: m1.update["uri"] = self.conf["rendezvous.server.update_uri"] m1.update["version"] = version # # Select test server address. # The default test server is the master server itself. # If we know the country, lookup the list of servers for # that country in the database. # If there are no servers for that country, register # the master server for the country so that we can notice # we have new users and can take the proper steps to # deploy nearby servers. # server = self.conf.get("rendezvous.server.default", "master.neubot.org") LOG.debug("* default test server: %s" % server) agent_address = stream.peername[0] country = GEOLOCATOR.lookup_country(agent_address) if country: servers = table_geoloc.lookup_servers(DATABASE.connection(), country) if not servers: LOG.info("* learning new country: %s" % country) table_geoloc.insert_server(DATABASE.connection(), country, server) servers = [server] server = random.choice(servers) LOG.debug("* selected test server: %s" % server) if "speedtest" in m.accept: m1.available["speedtest"] = [ "http://%s/speedtest" % server ] if "bittorrent" in m.accept: m1.available["bittorrent"] = [ "http://%s:8000/" % server ] # # Neubot <=0.3.7 expects to receive an XML document while # newer Neubots want a JSON. I hope old clients will upgrade # pretty soon. # if m.version and LibVersion.compare(m.version, "0.3.7") >= 0: s = marshal.marshal_object(m1, "application/json") mimetype = "application/json" else: s = compat.adhoc_marshaller(m1) mimetype = "text/xml" stringio = StringIO.StringIO() stringio.write(s) stringio.seek(0) response = Message() response.compose(code="200", reason="Ok", mimetype=mimetype, body=stringio) stream.send_response(request, response)
import os.path import sys if __name__ == "__main__": sys.path.insert(0, ".") from neubot.config import CONFIG from neubot.log import LOG from neubot import utils try: import GeoIP except ImportError: LOG.error("Missing dependency: GeoIP") LOG.info("Please install GeoIP python wrappers, e.g.") LOG.info(" sudo apt-get install python-geoip") sys.exit(1) COUNTRY_DATABASE = "/usr/local/share/GeoIP/GeoIP.dat" class Geolocator(object): def __init__(self): self.countries = None def open_or_die(self): path = CONFIG.get("rendezvous.geoip_wrapper.country_database", COUNTRY_DATABASE) # # Detect the common error case, i.e. that the user has
def main(args): config.register_descriptions() common.main("bittorrent", "Neubot BitTorrent module", args) conf = CONFIG.copy() config.finalize_conf(conf) if conf["bittorrent.listen"]: # # If we need to negotiate and we're runing # standalone we also need to bring up the # global HTTP server. # if conf["bittorrent.negotiate"]: HTTP_SERVER.configure(conf) HTTP_SERVER.listen((conf["bittorrent.address"], conf["bittorrent.negotiate.port"])) conf["negotiate.listen"] = True negotiate.run(POLLER, conf) # # Drop privileges after listen() so we can # bind() to privileged ports # if conf["bittorrent.daemonize"]: system.change_dir() system.go_background() LOG.redirect() system.drop_privileges(LOG.error) else: # # If possible use the runner, which will execute the # test in the context of the neubot daemon. Then exit # to bypass the run() invokation that is below here. # If the runner fails, fallback to the usual code path, # which executes the test in the context of the local # process. # Set 'runned.enabled' to 0 to bypass the runner and # run the test locally. # if (utils.intify(conf['runner.enabled']) and runner_clnt.runner_client(conf["agent.api.address"], conf["agent.api.port"], LOG.noisy, "bittorrent")): sys.exit(0) LOG.info('Will run the test in the local context...') # # When we're connecting to a remote host to perform a test # we want Neubot to quit at the end of the test. When this # happens the test code publishes the "testdone" event, so # here we prepare to intercept the event and break our main # loop. # NOTIFIER.subscribe("testdone", lambda event, ctx: POLLER.break_loop()) run(POLLER, conf) POLLER.loop()
logging.info("INFO w/ logging.info") # The following should work because it should not interpolate logging.debug("DEBUG w/ logging.debug", "ciao") logging.warning("WARNING w/ logging.warning") logging.error("ERROR w/ logging.error") LOG.verbose() logging.info("INFO w/ logging.info") logging.debug("DEBUG w/ logging.debug") logging.warning("WARNING w/ logging.warning") logging.error("ERROR w/ logging.error") LOG.error("testing neubot logger -- This is an error message") LOG.warning("testing neubot logger -- This is an warning message") LOG.info("testing neubot logger -- This is an info message") LOG.debug("testing neubot logger -- This is a debug message") print compat.json.dumps(LOG.listify()) try: raise Exception("Testing LOG.exception") except (KeyboardInterrupt, SystemExit): raise except: LOG.exception() LOG.exception(func=LOG.warning) LOG.start("Testing the in-progress feature") LOG.progress("...") LOG.progress() LOG.complete("success!")
def main(args): """ Starts the server module """ # # Register descriptions in main() only so that # we don't advertise the name of knobs that aren't # relevant in the current context. # CONFIG.register_descriptions({ "server.bittorrent": "Start up BitTorrent test and negotiate server", "server.daemonize": "Become a daemon and run in background", 'server.debug': 'Run the localhost-only debug server', "server.negotiate": "Turn on negotiation infrastructure", "server.rendezvous": "Start up rendezvous server", "server.sapi": "Turn on Server-side API", "server.speedtest": "Start up Speedtest test and negotiate server", }) common.main("server", "Neubot server-side component", args) conf = CONFIG.copy() # # Configure our global HTTP server and make # sure that we don't provide filesystem access # even by mistake. # conf["http.server.rootdir"] = "" HTTP_SERVER.configure(conf) # # New-style modules are started just setting a # bunch of conf[] variables and then invoking # their run() method in order to kick them off. # if conf["server.negotiate"]: negotiate.run(POLLER, conf) if conf["server.bittorrent"]: conf["bittorrent.listen"] = True conf["bittorrent.negotiate"] = True bittorrent.run(POLLER, conf) if conf['server.speedtest']: #conf['speedtest.listen'] = 1 # Not yet #conf['speedtest.negotiate'] = 1 # Not yet neubot.speedtest.wrapper.run(POLLER, conf) # Migrating from old style to new style if conf["server.rendezvous"]: #conf["rendezvous.listen"] = True # Not yet neubot.rendezvous.server.run(POLLER, conf) # # Historically Neubot runs on port 9773 and # 8080 but we would like to switch to port 80 # in the long term period, because it's rare # that they filter it. # OTOH it looks like it's not possible to # do that easily w/ M-Lab because the port # is already taken. # address = "0.0.0.0" ports = (80, 8080, 9773) for port in ports: HTTP_SERVER.listen((address, port)) # # Start server-side API for Nagios plugin # to query the state of the server. # functionalities. # if conf["server.sapi"]: server = ServerSideAPI(POLLER) server.configure(conf) HTTP_SERVER.register_child(server, "/sapi") # # Create localhost-only debug server # if CONFIG['server.debug']: LOG.info('server: Starting debug server at 127.0.0.1:9774') server = DebugAPI(POLLER) server.configure(conf) server.listen(('127.0.0.1', 9774)) # # Go background and drop privileges, # then enter into the main loop. # if conf["server.daemonize"]: system.change_dir() system.go_background() system.write_pidfile() LOG.redirect() system.drop_privileges(LOG.error) POLLER.loop()
def got_response(self, stream, request, response): if response.code != "200": LOG.complete("bad response") self._schedule() else: LOG.complete() s = response.body.read() try: m1 = marshal.unmarshal_object(s, "application/json", compat.RendezvousResponse) except ValueError: LOG.exception() self._schedule() else: if "version" in m1.update and "uri" in m1.update: ver, uri = m1.update["version"], m1.update["uri"] LOG.info("Version %s available at %s" % (ver, uri)) STATE.update("update", {"version": ver, "uri": uri}) # # Choose the test we would like to run even if # we're not going to run it because we're running # in debug mode or tests are disabled. # This allows us to print to the logger the test # we /would/ have choosen if we were allowed to run # it. # tests = [] if "speedtest" in m1.available: tests.append("speedtest") if "bittorrent" in m1.available: tests.append("bittorrent") #XXX alternate the two tests if self._latest: tests.remove(self._latest) test = random.choice(tests) self._latest = test LOG.info("* Chosen test: %s" % test) # Are we allowed to run a test? if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]: LOG.info("Tests are disabled... not running") self._schedule() else: if (CONFIG["privacy.informed"] and not CONFIG["privacy.can_collect"]): LOG.warning("cannot run test without permission " "to save the results") self._schedule() else: conf = self.conf.copy() # # Subscribe _before_ connecting. This way we # immediately see "testdone" if the connection fails # and we can _schedule the next attempt. # NOTIFIER.subscribe("testdone", lambda *a, **kw: \ self._schedule()) if test == "speedtest": conf["speedtest.client.uri"] = m1.available[ "speedtest"][0] client = ClientSpeedtest(POLLER) client.configure(conf) client.connect_uri() elif test == "bittorrent": conf["bittorrent._uri"] = m1.available[ "bittorrent"][0] bittorrent.run(POLLER, conf) else: NOTIFIER.publish("testdone")
def process_request(self, stream, request): m = marshal.unmarshal_object(request.body.read(), "application/xml", compat.RendezvousRequest) m1 = compat.RendezvousResponse() # # If we don't say anything the rendezvous server is not # going to prompt for updates. We need to specify the # updated version number explicitly when we start it up. # This should guarantee that we do not advertise -rc # releases and other weird things. # version = self.conf["rendezvous.server.update_version"] if version and m.version: diff = LibVersion.compare(version, m.version) LOG.debug('rendezvous: version=%s m.version=%s diff=%f' % ( version, m.version, diff)) if diff > 0: m1.update["uri"] = self.conf["rendezvous.server.update_uri"] m1.update["version"] = version # # Select test server address. # The default test server is the master server itself. # If we know the country, lookup the list of servers for # that country in the database. # We only redirect to other servers clients that have # agreed to give us the permission to publish, in order # to be compliant with M-Lab policy. # If there are no servers for that country, register # the master server for the country so that we can notice # we have new users and can take the proper steps to # deploy nearby servers. # server = self.conf.get("rendezvous.server.default", "master.neubot.org") LOG.debug("* default test server: %s" % server) # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # request_body = m.__dict__.copy() if 'privacy_can_share' in request_body: request_body['privacy_can_publish'] = request_body[ 'privacy_can_share'] del request_body['privacy_can_share'] # Redirect IFF have ALL privacy permissions if privacy.count_valid(request_body, 'privacy_') == 3: agent_address = stream.peername[0] country = GEOLOCATOR.lookup_country(agent_address) if country: servers = table_geoloc.lookup_servers(DATABASE.connection(), country) if not servers: LOG.info("* learning new country: %s" % country) table_geoloc.insert_server(DATABASE.connection(), country, server) servers = [server] server = random.choice(servers) LOG.debug("* selected test server: %s" % server) if "speedtest" in m.accept: m1.available["speedtest"] = [ "http://%s/speedtest" % server ] if "bittorrent" in m.accept: m1.available["bittorrent"] = [ "http://%s/" % server ] # # Neubot <=0.3.7 expects to receive an XML document while # newer Neubots want a JSON. I hope old clients will upgrade # pretty soon. # if m.version and LibVersion.compare(m.version, "0.3.7") >= 0: s = marshal.marshal_object(m1, "application/json") mimetype = "application/json" else: s = compat.adhoc_marshaller(m1) mimetype = "text/xml" stringio = StringIO.StringIO() stringio.write(s) stringio.seek(0) response = Message() response.compose(code="200", reason="Ok", mimetype=mimetype, body=stringio) stream.send_response(request, response)
def configure(self, conf, measurer=None): if measurer: LOG.info("* speedtest: ignoring upstream measurer") ClientHTTP.configure(self, conf, self.measurer)
def _sample_queue_length(self, *args, **kwargs): LOG.info("SessionTracker: queue length: %d\n" % len(self.queue)) self.task = POLLER.sched(60, self._sample_queue_length)