def do_collect(self, stream, request):
''' Invoked on GET /speedtest/collect '''
stream.response_rewriter = self._rewrite_response
request.uri = '/collect/speedtest'
xmlreq = marshal.unmarshal_object(request.body.read(),
'application/xml', SpeedtestCollect)
message = {
'uuid': xmlreq.client,
'timestamp': int(float(xmlreq.timestamp)), # old clients bug
'internal_address': xmlreq.internalAddress,
'real_address': xmlreq.realAddress,
'remote_address': xmlreq.remoteAddress,
'connect_time': xmlreq.connectTime,
'latency': xmlreq.latency,
'download_speed': xmlreq.downloadSpeed,
'upload_speed': xmlreq.uploadSpeed,
'privacy_informed': xmlreq.privacy_informed,
'privacy_can_collect': xmlreq.privacy_can_collect,
'privacy_can_share': xmlreq.privacy_can_share,
'platform': xmlreq.platform,
'neubot_version': xmlreq.neubot_version,
}
# XXX Here we don't rewrite content-length which becomes bogus
request['content-type'] = 'application/json'
request.body = StringIO.StringIO(json.dumps(message))
NEGOTIATE_SERVER.process_request(stream, request)
def got_response(self, stream, request, response):
m = marshal.unmarshal_object(response.body.read(), "text/xml", SpeedtestNegotiate_Response)
self.conf["speedtest.client.authorization"] = m.authorization
self.conf["speedtest.client.public_address"] = m.publicAddress
self.conf["speedtest.client.unchoked"] = utils.intify(m.unchoked)
if m.queuePos:
self.conf["speedtest.client.queuepos"] = m.queuePos
def migrate_from__v1_1__to__v2_0(connection):
cursor = connection.cursor()
cursor.execute("SELECT value FROM config WHERE name='version';")
ver = cursor.fetchone()[0]
if ver == "1.1":
logging.info("* Migrating database from version 1.1 to 2.0")
connection.execute(
"""CREATE TABLE IF NOT EXISTS speedtest(
id INTEGER PRIMARY KEY,
timestamp INTEGER,
uuid TEXT,
internal_address TEXT,
real_address TEXT,
remote_address TEXT,
connect_time NUMERIC,
latency NUMERIC,
download_speed NUMERIC,
upload_speed NUMERIC,
privacy_informed NUMERIC,
privacy_can_collect NUMERIC,
privacy_can_share NUMERIC
);"""
)
query = """INSERT INTO speedtest VALUES (
null,
:timestamp,
:uuid,
:internal_address,
:real_address,
:remote_address,
:connect_time,
:latency,
:download_speed,
:upload_speed,
:privacy_informed,
:privacy_can_collect,
:privacy_can_share
);"""
cursor.execute("SELECT result, timestamp, uuid FROM results ORDER BY timestamp;")
for result, timestamp, uuid in cursor:
result = unmarshal_object(result, "application/xml", SpeedtestResultXML)
result = speedtest_result_good_from_xml(result)
result["timestamp"] = timestamp
result["uuid"] = uuid
result["privacy_informed"] = 0
result["privacy_can_collect"] = 0
result["privacy_can_share"] = 0
connection.execute(query, result)
connection.execute("DROP TABLE results;")
connection.execute(
"""UPDATE config SET value='2.0'
WHERE name='version';"""
)
connection.commit()
def got_response(self, stream, request, response):
if response.code != "200":
LOG.complete("bad response")
self._schedule()
else:
LOG.complete()
s = response.body.read()
try:
m1 = marshal.unmarshal_object(s, "application/json", compat.RendezvousResponse)
except ValueError:
LOG.exception()
self._schedule()
else:
if "version" in m1.update and "uri" in m1.update:
ver, uri = m1.update["version"], m1.update["uri"]
LOG.info("Version %s available at %s" % (ver, uri))
STATE.update("update", {"version": ver, "uri": uri})
_open_browser_on_windows("update.html")
# Update tests known by the runner
runner_lst.update(m1.available)
#
# Choose the test we would like to run even if
# we're not going to run it because we're running
# in debug mode or tests are disabled.
# This allows us to print to the logger the test
# we /would/ have choosen if we were allowed to run
# it.
#
test = runner_lst.get_next_test()
if not test:
LOG.warning("No test available")
self._schedule()
return
LOG.info("* Chosen test: %s" % test)
# Are we allowed to run a test?
if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]:
LOG.info("Tests are disabled... not running")
self._schedule()
else:
# Do we have negotiate URI for test?
negotiate_uri = runner_lst.test_to_negotiate_uri(test)
if not negotiate_uri:
LOG.warning("No negotiate URI for test")
self._schedule()
else:
# Actually run the test
runner_core.run(test, negotiate_uri, self._schedule)
def do_collect(self, stream, request):
self._speedtest_complete(request)
s = request.body.read()
m = marshal.unmarshal_object(s, "text/xml", compat.SpeedtestCollect)
if privacy.collect_allowed(m):
table_speedtest.insertxxx(DATABASE.connection(), m)
response = Message()
response.compose(code="200", reason="Ok")
stream.send_response(request, response)
def process_request(self, stream, request):
m = marshal.unmarshal_object(request.body.read(),
"application/xml", compat.RendezvousRequest)
m1 = compat.RendezvousResponse()
version = self.conf["rendezvous.server.update_version"]
#
# Don't offer a release candidate update if the user is not
# running a release candidate as well and viceversa.
#
if (("-rc" in version and "-rc" in m.version) or
(not "-rc" in version and not "-rc" in m.version)):
if m.version and LibVersion.compare(version, m.version) > 0:
m1.update["uri"] = self.conf["rendezvous.server.update_uri"]
m1.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default",
"master.neubot.org")
LOG.debug("* default test server: %s" % server)
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(),
country)
if not servers:
LOG.info("* learning new country: %s" % country)
table_geoloc.insert_server(DATABASE.connection(),
country, server)
servers = [server]
server = random.choice(servers)
LOG.debug("* selected test server: %s" % server)
if "speedtest" in m.accept:
m1.available["speedtest"] = [ "http://%s/speedtest" % server ]
if "bittorrent" in m.accept:
m1.available["bittorrent"] = [ "http://%s:8000/" % server ]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if m.version and LibVersion.compare(m.version, "0.3.7") >= 0:
s = marshal.marshal_object(m1, "application/json")
mimetype = "application/json"
else:
s = compat.adhoc_marshaller(m1)
mimetype = "text/xml"
stringio = StringIO.StringIO()
stringio.write(s)
stringio.seek(0)
response = Message()
response.compose(code="200", reason="Ok",
mimetype=mimetype, body=stringio)
stream.send_response(request, response)
def got_response(self, stream, request, response):
if response.code != "200":
LOG.complete("bad response")
self._schedule()
else:
LOG.complete()
s = response.body.read()
try:
m1 = marshal.unmarshal_object(s, "application/json",
compat.RendezvousResponse)
except ValueError:
LOG.exception()
self._schedule()
else:
if "version" in m1.update and "uri" in m1.update:
ver, uri = m1.update["version"], m1.update["uri"]
LOG.info("Version %s available at %s" % (ver, uri))
STATE.update("update", {"version": ver, "uri": uri})
#
# Choose the test we would like to run even if
# we're not going to run it because we're running
# in debug mode or tests are disabled.
# This allows us to print to the logger the test
# we /would/ have choosen if we were allowed to run
# it.
#
tests = []
if "speedtest" in m1.available:
tests.append("speedtest")
if "bittorrent" in m1.available:
tests.append("bittorrent")
#XXX alternate the two tests
if self._latest:
tests.remove(self._latest)
test = random.choice(tests)
self._latest = test
LOG.info("* Chosen test: %s" % test)
# Are we allowed to run a test?
if not CONFIG["enabled"] or CONFIG["rendezvous.client.debug"]:
LOG.info("Tests are disabled... not running")
self._schedule()
else:
if (CONFIG["privacy.informed"] and
not CONFIG["privacy.can_collect"]):
LOG.warning("cannot run test without permission "
"to save the results")
self._schedule()
else:
conf = self.conf.copy()
#
# Subscribe _before_ connecting. This way we
# immediately see "testdone" if the connection fails
# and we can _schedule the next attempt.
#
NOTIFIER.subscribe("testdone", lambda *a, **kw: \
self._schedule())
if test == "speedtest":
conf["speedtest.client.uri"] = m1.available[
"speedtest"][0]
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
elif test == "bittorrent":
conf["bittorrent._uri"] = m1.available[
"bittorrent"][0]
bittorrent.run(POLLER, conf)
else:
NOTIFIER.publish("testdone")
def process_request(self, stream, request):
m = marshal.unmarshal_object(request.body.read(),
"application/xml", compat.RendezvousRequest)
m1 = compat.RendezvousResponse()
#
# If we don't say anything the rendezvous server is not
# going to prompt for updates. We need to specify the
# updated version number explicitly when we start it up.
# This should guarantee that we do not advertise -rc
# releases and other weird things.
#
version = self.conf["rendezvous.server.update_version"]
if version and m.version:
diff = LibVersion.compare(version, m.version)
LOG.debug('rendezvous: version=%s m.version=%s diff=%f' % (
version, m.version, diff))
if diff > 0:
m1.update["uri"] = self.conf["rendezvous.server.update_uri"]
m1.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# We only redirect to other servers clients that have
# agreed to give us the permission to publish, in order
# to be compliant with M-Lab policy.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default",
"master.neubot.org")
LOG.debug("* default test server: %s" % server)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
request_body = m.__dict__.copy()
if 'privacy_can_share' in request_body:
request_body['privacy_can_publish'] = request_body[
'privacy_can_share']
del request_body['privacy_can_share']
# Redirect IFF have ALL privacy permissions
if privacy.count_valid(request_body, 'privacy_') == 3:
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(),
country)
if not servers:
LOG.info("* learning new country: %s" % country)
table_geoloc.insert_server(DATABASE.connection(),
country, server)
servers = [server]
server = random.choice(servers)
LOG.debug("* selected test server: %s" % server)
if "speedtest" in m.accept:
m1.available["speedtest"] = [ "http://%s/speedtest" % server ]
if "bittorrent" in m.accept:
m1.available["bittorrent"] = [ "http://%s/" % server ]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if m.version and LibVersion.compare(m.version, "0.3.7") >= 0:
s = marshal.marshal_object(m1, "application/json")
mimetype = "application/json"
else:
s = compat.adhoc_marshaller(m1)
mimetype = "text/xml"
stringio = StringIO.StringIO()
stringio.write(s)
stringio.seek(0)
response = Message()
response.compose(code="200", reason="Ok",
mimetype=mimetype, body=stringio)
stream.send_response(request, response)
def process_request(self, stream, request):
""" Process rendezvous request """
if request["content-type"] == "application/json":
ibody = marshal.unmarshal_object(request.body.read(), "application/json", compat.RendezvousRequest)
else:
ibody = marshal.unmarshal_object(request.body.read(), "application/xml", compat.RendezvousRequest)
obody = compat.RendezvousResponse()
#
# If we don't say anything the rendezvous server is not
# going to prompt for updates. We need to specify the
# updated version number explicitly when we start it up.
# This should guarantee that we do not advertise -rc
# releases and other weird things.
#
version = self.conf["rendezvous.server.update_version"]
if version and ibody.version:
diff = utils_version.compare(version, ibody.version)
logging.debug("rendezvous: version=%s ibody.version=%s diff=%f", version, ibody.version, diff)
if diff > 0:
obody.update["uri"] = "http://neubot.org/"
obody.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# We only redirect to other servers clients that have
# agreed to give us the permission to publish, in order
# to be compliant with M-Lab policy.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default", "master.neubot.org")
logging.debug("* default test server: %s", server)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
request_body = ibody.__dict__.copy()
if "privacy_can_share" in request_body:
request_body["privacy_can_publish"] = request_body["privacy_can_share"]
del request_body["privacy_can_share"]
# Redirect IFF have ALL privacy permissions
if privacy.count_valid(request_body, "privacy_") == 3:
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(), country)
if not servers:
logging.info("* learning new country: %s", country)
table_geoloc.insert_server(DATABASE.connection(), country, server)
servers = [server]
server = random.choice(servers)
logging.info("rendezvous_server: %s[%s] -> %s", agent_address, country, server)
else:
logging.warning("rendezvous_server: cannot redirect to M-Lab: %s", request_body)
#
# We require at least informed and can_collect since 0.4.4
# (released 25 October 2011), so stop clients with empty
# privacy settings, who were still using master.
#
if privacy.collect_allowed(request_body):
#
# Note: Here we will have problems if we store unquoted
# IPv6 addresses into the database. Because the resulting
# URI won't be valid.
#
if "speedtest" in ibody.accept:
obody.available["speedtest"] = ["http://%s/speedtest" % server]
if "bittorrent" in ibody.accept:
obody.available["bittorrent"] = ["http://%s/" % server]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if ibody.version and utils_version.compare(ibody.version, "0.3.7") >= 0:
body = marshal.marshal_object(obody, "application/json")
mimetype = "application/json"
else:
body = compat.adhoc_marshaller(obody)
mimetype = "text/xml"
response = Message()
response.compose(code="200", reason="Ok", mimetype=mimetype, body=body)
stream.send_response(request, response)
