def main(args):
CONFIG.register_descriptions({
"speedtest.client.uri": "Base URI to connect to",
"speedtest.client.nconn": "Number of concurrent connections to use",
"speedtest.client.latency_tries": "Number of latency measurements",
})
common.main("speedtest.client", "Speedtest client", args)
conf = CONFIG.copy()
#
# If possible use the runner, which will execute the
# test in the context of the neubot daemon. Then exit
# to bypass the POLLER.loop() invokation that is below
# here.
# If the runner fails, fallback to the usual code path,
# which executes the test in the context of the local
# process.
# Set 'runned.enabled' to 0 to bypass the runner and
# run the test locally.
#
if (utils.intify(conf['runner.enabled']) and
runner_clnt.runner_client(conf["agent.api.address"],
conf["agent.api.port"],
LOG.noisy, "speedtest")):
sys.exit(0)
LOG.info('Will run the test in the local context...')
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def connect_uri(self, uri=None, count=None):
if not uri:
uri = self.conf.get("speedtest.client.uri",
"http://master.neubot.org/")
if not count:
count = self.conf.get("speedtest.client.nconn", 1)
LOG.info("* speedtest with %s" % uri)
ClientHTTP.connect_uri(self, uri, count)
def run(poller, conf):
""" Load MaxMind database and register our child server """
GEOLOCATOR.open_or_die()
LOG.info("This product includes GeoLite data created by MaxMind, "
"available from <http://www.maxmind.com/>.")
server = ServerRendezvous(None)
server.configure(conf)
HTTP_SERVER.register_child(server, "/rendezvous")
def run(self, test, callback, auto_rendezvous=True, ctx=None):
''' Run test and callback() when done '''
#
# If we are about to run a test and the list of
# available tests is empty, we need certainly to
# refill it before proceeding.
#
if (auto_rendezvous and test != 'rendezvous' and
len(RUNNER_TESTS.get_test_names()) == 0):
LOG.info('runner_core: Need to rendezvous first...')
self.queue.append(('rendezvous', lambda: None, None))
self.queue.append((test, callback, ctx))
self.run_queue()
def open_or_die(self):
path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
COUNTRY_DATABASE)
#
# Detect the common error case, i.e. that the user has
# not downloaded the database. If something fancy is
# going on, let the GeoIP library stacktrace for us.
#
if not os.path.exists(path):
LOG.error("Missing GeoLiteCountry database: %s" % path)
LOG.info("Please download it from "
"<http://www.maxmind.com/app/geolitecountry>.")
sys.exit(1)
self.countries = GeoIP.open(path, GeoIP.GEOIP_STANDARD)
def _schedule(self):
#
# Typically the user does not specify the interval
# and we use a random value around 1500 seconds.
# The random value is extracted just once and from
# that point on we keep using it.
#
interval = CONFIG["agent.interval"]
if not interval:
if not self._interval:
self._interval = 1380 + random.randrange(0, 240)
interval = self._interval
LOG.info("* Next rendezvous in %d seconds" % interval)
fn = lambda *args, **kwargs: self.connect_uri()
self._task = POLLER.sched(interval, fn)
STATE.update("idle", publish=False)
STATE.update("next_rendezvous", self._task.timestamp)
def got_response(self, stream, request, response):
if response.code != "200":
LOG.complete("bad response")
self._schedule()
else:
LOG.complete()
s = response.body.read()
try:
m1 = marshal.unmarshal_object(s, "application/json", compat.RendezvousResponse)
except ValueError:
LOG.exception()
self._schedule()
else:
if "version" in m1.update and "uri" in m1.update:
ver, uri = m1.update["version"], m1.update["uri"]
LOG.info("Version %s available at %s" % (ver, uri))
STATE.update("update", {"version": ver, "uri": uri})
_open_browser_on_windows("update.html")
# Update tests known by the runner
runner_lst.update(m1.available)
#
# Choose the test we would like to run even if
# we're not going to run it because we're running
# in debug mode or tests are disabled.
# This allows us to print to the logger the test
# we /would/ have choosen if we were allowed to run
# it.
#
test = runner_lst.get_next_test()
if not test:
LOG.warning("No test available")
self._schedule()
return
LOG.info("* Chosen test: %s" % test)
# Are we allowed to run a test?
if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]:
LOG.info("Tests are disabled... not running")
self._schedule()
else:
# Do we have negotiate URI for test?
negotiate_uri = runner_lst.test_to_negotiate_uri(test)
if not negotiate_uri:
LOG.warning("No negotiate URI for test")
self._schedule()
else:
# Actually run the test
runner_core.run(test, negotiate_uri, self._schedule)
def process_request(self, stream, request):
m = marshal.unmarshal_object(request.body.read(),
"application/xml", compat.RendezvousRequest)
m1 = compat.RendezvousResponse()
version = self.conf["rendezvous.server.update_version"]
#
# Don't offer a release candidate update if the user is not
# running a release candidate as well and viceversa.
#
if (("-rc" in version and "-rc" in m.version) or
(not "-rc" in version and not "-rc" in m.version)):
if m.version and LibVersion.compare(version, m.version) > 0:
m1.update["uri"] = self.conf["rendezvous.server.update_uri"]
m1.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default",
"master.neubot.org")
LOG.debug("* default test server: %s" % server)
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(),
country)
if not servers:
LOG.info("* learning new country: %s" % country)
table_geoloc.insert_server(DATABASE.connection(),
country, server)
servers = [server]
server = random.choice(servers)
LOG.debug("* selected test server: %s" % server)
if "speedtest" in m.accept:
m1.available["speedtest"] = [ "http://%s/speedtest" % server ]
if "bittorrent" in m.accept:
m1.available["bittorrent"] = [ "http://%s:8000/" % server ]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if m.version and LibVersion.compare(m.version, "0.3.7") >= 0:
s = marshal.marshal_object(m1, "application/json")
mimetype = "application/json"
else:
s = compat.adhoc_marshaller(m1)
mimetype = "text/xml"
stringio = StringIO.StringIO()
stringio.write(s)
stringio.seek(0)
response = Message()
response.compose(code="200", reason="Ok",
mimetype=mimetype, body=stringio)
stream.send_response(request, response)
import os.path
import sys
if __name__ == "__main__":
sys.path.insert(0, ".")
from neubot.config import CONFIG
from neubot.log import LOG
from neubot import utils
try:
import GeoIP
except ImportError:
LOG.error("Missing dependency: GeoIP")
LOG.info("Please install GeoIP python wrappers, e.g.")
LOG.info(" sudo apt-get install python-geoip")
sys.exit(1)
COUNTRY_DATABASE = "/usr/local/share/GeoIP/GeoIP.dat"
class Geolocator(object):
def __init__(self):
self.countries = None
def open_or_die(self):
path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
COUNTRY_DATABASE)
#
# Detect the common error case, i.e. that the user has
def main(args):
config.register_descriptions()
common.main("bittorrent", "Neubot BitTorrent module", args)
conf = CONFIG.copy()
config.finalize_conf(conf)
if conf["bittorrent.listen"]:
#
# If we need to negotiate and we're runing
# standalone we also need to bring up the
# global HTTP server.
#
if conf["bittorrent.negotiate"]:
HTTP_SERVER.configure(conf)
HTTP_SERVER.listen((conf["bittorrent.address"],
conf["bittorrent.negotiate.port"]))
conf["negotiate.listen"] = True
negotiate.run(POLLER, conf)
#
# Drop privileges after listen() so we can
# bind() to privileged ports
#
if conf["bittorrent.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
else:
#
# If possible use the runner, which will execute the
# test in the context of the neubot daemon. Then exit
# to bypass the run() invokation that is below here.
# If the runner fails, fallback to the usual code path,
# which executes the test in the context of the local
# process.
# Set 'runned.enabled' to 0 to bypass the runner and
# run the test locally.
#
if (utils.intify(conf['runner.enabled']) and
runner_clnt.runner_client(conf["agent.api.address"],
conf["agent.api.port"],
LOG.noisy, "bittorrent")):
sys.exit(0)
LOG.info('Will run the test in the local context...')
#
# When we're connecting to a remote host to perform a test
# we want Neubot to quit at the end of the test. When this
# happens the test code publishes the "testdone" event, so
# here we prepare to intercept the event and break our main
# loop.
#
NOTIFIER.subscribe("testdone", lambda event, ctx: POLLER.break_loop())
run(POLLER, conf)
POLLER.loop()
logging.info("INFO w/ logging.info")
# The following should work because it should not interpolate
logging.debug("DEBUG w/ logging.debug", "ciao")
logging.warning("WARNING w/ logging.warning")
logging.error("ERROR w/ logging.error")
LOG.verbose()
logging.info("INFO w/ logging.info")
logging.debug("DEBUG w/ logging.debug")
logging.warning("WARNING w/ logging.warning")
logging.error("ERROR w/ logging.error")
LOG.error("testing neubot logger -- This is an error message")
LOG.warning("testing neubot logger -- This is an warning message")
LOG.info("testing neubot logger -- This is an info message")
LOG.debug("testing neubot logger -- This is a debug message")
print compat.json.dumps(LOG.listify())
try:
raise Exception("Testing LOG.exception")
except (KeyboardInterrupt, SystemExit):
raise
except:
LOG.exception()
LOG.exception(func=LOG.warning)
LOG.start("Testing the in-progress feature")
LOG.progress("...")
LOG.progress()
LOG.complete("success!")
def main(args):
""" Starts the server module """
#
# Register descriptions in main() only so that
# we don't advertise the name of knobs that aren't
# relevant in the current context.
#
CONFIG.register_descriptions({
"server.bittorrent": "Start up BitTorrent test and negotiate server",
"server.daemonize": "Become a daemon and run in background",
'server.debug': 'Run the localhost-only debug server',
"server.negotiate": "Turn on negotiation infrastructure",
"server.rendezvous": "Start up rendezvous server",
"server.sapi": "Turn on Server-side API",
"server.speedtest": "Start up Speedtest test and negotiate server",
})
common.main("server", "Neubot server-side component", args)
conf = CONFIG.copy()
#
# Configure our global HTTP server and make
# sure that we don't provide filesystem access
# even by mistake.
#
conf["http.server.rootdir"] = ""
HTTP_SERVER.configure(conf)
#
# New-style modules are started just setting a
# bunch of conf[] variables and then invoking
# their run() method in order to kick them off.
#
if conf["server.negotiate"]:
negotiate.run(POLLER, conf)
if conf["server.bittorrent"]:
conf["bittorrent.listen"] = True
conf["bittorrent.negotiate"] = True
bittorrent.run(POLLER, conf)
if conf['server.speedtest']:
#conf['speedtest.listen'] = 1 # Not yet
#conf['speedtest.negotiate'] = 1 # Not yet
neubot.speedtest.wrapper.run(POLLER, conf)
# Migrating from old style to new style
if conf["server.rendezvous"]:
#conf["rendezvous.listen"] = True # Not yet
neubot.rendezvous.server.run(POLLER, conf)
#
# Historically Neubot runs on port 9773 and
# 8080 but we would like to switch to port 80
# in the long term period, because it's rare
# that they filter it.
# OTOH it looks like it's not possible to
# do that easily w/ M-Lab because the port
# is already taken.
#
address = "0.0.0.0"
ports = (80, 8080, 9773)
for port in ports:
HTTP_SERVER.listen((address, port))
#
# Start server-side API for Nagios plugin
# to query the state of the server.
# functionalities.
#
if conf["server.sapi"]:
server = ServerSideAPI(POLLER)
server.configure(conf)
HTTP_SERVER.register_child(server, "/sapi")
#
# Create localhost-only debug server
#
if CONFIG['server.debug']:
LOG.info('server: Starting debug server at 127.0.0.1:9774')
server = DebugAPI(POLLER)
server.configure(conf)
server.listen(('127.0.0.1', 9774))
#
# Go background and drop privileges,
# then enter into the main loop.
#
if conf["server.daemonize"]:
system.change_dir()
system.go_background()
system.write_pidfile()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def got_response(self, stream, request, response):
if response.code != "200":
LOG.complete("bad response")
self._schedule()
else:
LOG.complete()
s = response.body.read()
try:
m1 = marshal.unmarshal_object(s, "application/json",
compat.RendezvousResponse)
except ValueError:
LOG.exception()
self._schedule()
else:
if "version" in m1.update and "uri" in m1.update:
ver, uri = m1.update["version"], m1.update["uri"]
LOG.info("Version %s available at %s" % (ver, uri))
STATE.update("update", {"version": ver, "uri": uri})
#
# Choose the test we would like to run even if
# we're not going to run it because we're running
# in debug mode or tests are disabled.
# This allows us to print to the logger the test
# we /would/ have choosen if we were allowed to run
# it.
#
tests = []
if "speedtest" in m1.available:
tests.append("speedtest")
if "bittorrent" in m1.available:
tests.append("bittorrent")
#XXX alternate the two tests
if self._latest:
tests.remove(self._latest)
test = random.choice(tests)
self._latest = test
LOG.info("* Chosen test: %s" % test)
# Are we allowed to run a test?
if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]:
LOG.info("Tests are disabled... not running")
self._schedule()
else:
if (CONFIG["privacy.informed"] and
not CONFIG["privacy.can_collect"]):
LOG.warning("cannot run test without permission "
"to save the results")
self._schedule()
else:
conf = self.conf.copy()
#
# Subscribe _before_ connecting. This way we
# immediately see "testdone" if the connection fails
# and we can _schedule the next attempt.
#
NOTIFIER.subscribe("testdone", lambda *a, **kw: \
self._schedule())
if test == "speedtest":
conf["speedtest.client.uri"] = m1.available[
"speedtest"][0]
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
elif test == "bittorrent":
conf["bittorrent._uri"] = m1.available[
"bittorrent"][0]
bittorrent.run(POLLER, conf)
else:
NOTIFIER.publish("testdone")
def process_request(self, stream, request):
m = marshal.unmarshal_object(request.body.read(),
"application/xml", compat.RendezvousRequest)
m1 = compat.RendezvousResponse()
#
# If we don't say anything the rendezvous server is not
# going to prompt for updates. We need to specify the
# updated version number explicitly when we start it up.
# This should guarantee that we do not advertise -rc
# releases and other weird things.
#
version = self.conf["rendezvous.server.update_version"]
if version and m.version:
diff = LibVersion.compare(version, m.version)
LOG.debug('rendezvous: version=%s m.version=%s diff=%f' % (
version, m.version, diff))
if diff > 0:
m1.update["uri"] = self.conf["rendezvous.server.update_uri"]
m1.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# We only redirect to other servers clients that have
# agreed to give us the permission to publish, in order
# to be compliant with M-Lab policy.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default",
"master.neubot.org")
LOG.debug("* default test server: %s" % server)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
request_body = m.__dict__.copy()
if 'privacy_can_share' in request_body:
request_body['privacy_can_publish'] = request_body[
'privacy_can_share']
del request_body['privacy_can_share']
# Redirect IFF have ALL privacy permissions
if privacy.count_valid(request_body, 'privacy_') == 3:
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(),
country)
if not servers:
LOG.info("* learning new country: %s" % country)
table_geoloc.insert_server(DATABASE.connection(),
country, server)
servers = [server]
server = random.choice(servers)
LOG.debug("* selected test server: %s" % server)
if "speedtest" in m.accept:
m1.available["speedtest"] = [ "http://%s/speedtest" % server ]
if "bittorrent" in m.accept:
m1.available["bittorrent"] = [ "http://%s/" % server ]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if m.version and LibVersion.compare(m.version, "0.3.7") >= 0:
s = marshal.marshal_object(m1, "application/json")
mimetype = "application/json"
else:
s = compat.adhoc_marshaller(m1)
mimetype = "text/xml"
stringio = StringIO.StringIO()
stringio.write(s)
stringio.seek(0)
response = Message()
response.compose(code="200", reason="Ok",
mimetype=mimetype, body=stringio)
stream.send_response(request, response)
def configure(self, conf, measurer=None):
if measurer:
LOG.info("* speedtest: ignoring upstream measurer")
ClientHTTP.configure(self, conf, self.measurer)
def _sample_queue_length(self, *args, **kwargs):
LOG.info("SessionTracker: queue length: %d\n" % len(self.queue))
self.task = POLLER.sched(60, self._sample_queue_length)
