def test_success(self):
''' Make sure collect_allowed() returns True on valid input '''
self.assertTrue(privacy.collect_allowed(
{'privacy_informed': 1,
'privacy_can_collect': 1,
'privacy_can_publish': 0}))
self.assertTrue(privacy.collect_allowed(
{'privacy_informed': 1,
'privacy_can_collect': 1,
'privacy_can_publish': 1}))
def test_success(self):
''' Make sure collect_allowed() returns True on valid input '''
self.assertTrue(
privacy.collect_allowed({
'privacy_informed': 1,
'privacy_can_collect': 1,
'privacy_can_publish': 0
}))
self.assertTrue(
privacy.collect_allowed({
'privacy_informed': 1,
'privacy_can_collect': 1,
'privacy_can_publish': 1
}))
def collect(self, m):
btid = _make_btid(m["ident"])
if btid not in AUTH_PEERS:
raise NegotiatorEOF()
d = m["request_body"]
result = AUTH_PEERS[btid]
#
# Note that the following is not a bug: it's just that
# the server saves results using the point of view of the
# client, i.e. upload_speed _is_ client's upload speed.
#
d["timestamp"] = result["timestamp"]
d["upload_speed"] = result["upload_speed"]
if privacy.collect_allowed(d):
table_bittorrent.insert(DATABASE.connection(), d)
#
# After we've saved the result into the dictionary we
# can add extra information we would like to return to
# the client.
#
d["target_bytes"] = result["target_bytes"]
m["response_body"] = d
def got_response_collecting(self, stream, request, response):
LOG.complete()
if self.success:
#
# Always measure at the receiver because there is more
# information at the receiver and also to make my friend
# Enrico happier :-P.
# The following is not a bug: it's just that the server
# returns a result using the point of view of the client,
# i.e. upload_speed is _our_ upload speed.
#
m = json.loads(response.body.read())
self.my_side["upload_speed"] = m["upload_speed"]
upload = utils.speed_formatter(m["upload_speed"])
STATE.update("test_upload", upload)
if privacy.collect_allowed(self.my_side):
table_bittorrent.insert(DATABASE.connection(), self.my_side)
# Update the upstream channel estimate
target_bytes = int(m["target_bytes"])
if target_bytes > 0:
estimate.UPLOAD = target_bytes
stream.close()
def got_response_collecting(self, stream, request, response):
logging.info("BitTorrent: collecting ... done")
if self.success:
#
# Always measure at the receiver because there is more
# information at the receiver and also to make my friend
# Enrico happier :-P.
# The following is not a bug: it's just that the server
# returns a result using the point of view of the client,
# i.e. upload_speed is _our_ upload speed.
#
m = json.loads(response.body.read())
self.my_side["upload_speed"] = m["upload_speed"]
upload = utils.speed_formatter(m["upload_speed"])
STATE.update("test_progress", "100%", publish=False)
STATE.update("test_upload", upload)
logging.info('BitTorrent: upload speed: %s', upload)
if privacy.collect_allowed(self.my_side):
if DATABASE.readonly:
logging.warning('bittorrent_client: readonly database')
else:
table_bittorrent.insert(DATABASE.connection(), self.my_side)
# Update the upstream channel estimate
target_bytes = int(m["target_bytes"])
if target_bytes > 0:
estimate.UPLOAD = target_bytes
self.final_state = True
stream.close()
def do_collect(self, stream, request):
self._speedtest_complete(request)
s = request.body.read()
m = marshal.unmarshal_object(s, "text/xml", compat.SpeedtestCollect)
if privacy.collect_allowed(m):
table_speedtest.insertxxx(DATABASE.connection(), m)
response = Message()
response.compose(code="200", reason="Ok")
stream.send_response(request, response)
def test_failure(self):
''' Make sure collect_allowed() returns False on bad input '''
self.assertFalse(privacy.collect_allowed(
{'privacy_informed': 0,
'privacy_can_collect': 0,
'privacy_can_publish': 0}))
self.assertFalse(privacy.collect_allowed(
{'privacy_informed': 0,
'privacy_can_collect': 0,
'privacy_can_publish': 1}))
self.assertFalse(privacy.collect_allowed(
{'privacy_informed': 0,
'privacy_can_collect': 1,
'privacy_can_publish': 0}))
self.assertFalse(privacy.collect_allowed(
{'privacy_informed': 0,
'privacy_can_collect': 1,
'privacy_can_publish': 1}))
self.assertFalse(privacy.collect_allowed(
{'privacy_informed': 1,
'privacy_can_collect': 0,
'privacy_can_publish': 0}))
self.assertFalse(privacy.collect_allowed(
{'privacy_informed': 1,
'privacy_can_collect': 0,
'privacy_can_publish': 1}))
def connection_ready(self, stream):
m1 = SpeedtestCollect()
m1.client = self.conf.get("uuid", "")
m1.timestamp = utils.timestamp()
m1.internalAddress = stream.myname[0]
m1.realAddress = self.conf.get("speedtest.client.public_address", "")
m1.remoteAddress = stream.peername[0]
m1.latency = self.conf.get("speedtest.client.latency", 0.0)
m1.downloadSpeed = self.conf.get("speedtest.client.download", 0.0)
m1.uploadSpeed = self.conf.get("speedtest.client.upload", 0.0)
m1.privacy_informed = self.conf.get("privacy.informed", 0)
m1.privacy_can_collect = self.conf.get("privacy.can_collect", 0)
m1.privacy_can_share = self.conf.get("privacy.can_publish", 0) # XXX
m1.neubot_version = utils_version.NUMERIC_VERSION
m1.platform = sys.platform
m1.connectTime = sum(self.rtts) / len(self.rtts)
# Test version (added Neubot 0.4.12)
m1.testVersion = CONFIG["speedtest_test_version"]
s = marshal.marshal_object(m1, "text/xml")
stringio = StringIO.StringIO(s)
#
# Pass a dictionary because the function does not accept
# anymore an object
#
if privacy.collect_allowed(m1.__dict__):
if DATABASE.readonly:
logging.warning("speedtest: readonly database")
else:
insertxxx(DATABASE.connection(), m1)
request = Message()
request.compose(
method="POST",
pathquery="/speedtest/collect",
body=stringio,
mimetype="application/xml",
host=self.host_header,
)
request["authorization"] = self.conf.get("speedtest.client.authorization", "")
stream.send_request(request)
def connection_ready(self, stream):
m1 = compat.SpeedtestCollect()
m1.client = self.conf.get("uuid", "")
m1.timestamp = utils.timestamp()
m1.internalAddress = stream.myname[0]
m1.realAddress = self.conf.get("speedtest.client.public_address", "")
m1.remoteAddress = stream.peername[0]
m1.latency = self.conf.get("speedtest.client.latency", 0.0)
m1.downloadSpeed = self.conf.get("speedtest.client.download", 0.0)
m1.uploadSpeed = self.conf.get("speedtest.client.upload", 0.0)
m1.privacy_informed = self.conf.get("privacy.informed", 0)
m1.privacy_can_collect = self.conf.get("privacy.can_collect", 0)
m1.privacy_can_share = self.conf.get("privacy.can_share", 0)
m1.neubot_version = LibVersion.to_numeric("0.4.2")
m1.platform = sys.platform
if self.measurer:
m1.connectTime = self.measurer.measure_rtt()[0]
# import pprint
# if hasattr(self.measurer, "recv_hist"):
# download = self.measurer.recv_hist.get("download", [])
# pprint.pprint(download)
# if hasattr(self.measurer, "send_hist"):
# upload = self.measurer.send_hist.get("upload", [])
# pprint.pprint(upload)
s = marshal.marshal_object(m1, "text/xml")
stringio = StringIO.StringIO(s)
if privacy.collect_allowed(m1):
table_speedtest.insertxxx(DATABASE.connection(), m1)
request = Message()
request.compose(method="POST", pathquery="/speedtest/collect",
body=stringio, mimetype="application/xml",
host=self.host_header)
request["authorization"] = self.conf.get(
"speedtest.client.authorization", "")
stream.send_request(request)
def collect(self, stream, request_body):
''' Invoked when we must save the result of a session '''
sha1 = self._stream_to_sha1(stream)
if sha1 not in self.peers:
raise RuntimeError('Not authorized to collect')
else:
# Note: no more than one collect per session
result = self.peers[sha1]
del self.peers[sha1]
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
if 'privacy_can_share' in request_body:
request_body['privacy_can_publish'] = request_body[
'privacy_can_share']
del request_body['privacy_can_share']
#
# Note that the following is not a bug: it's just that
# the server saves results using the point of view of the
# client, i.e. upload_speed _is_ client's upload speed.
#
request_body['timestamp'] = result['timestamp']
request_body['upload_speed'] = result['upload_speed']
if privacy.collect_allowed(request_body):
BACKEND.bittorrent_store(request_body)
else:
logging.warning('* bad privacy settings: %s', str(stream))
#
# After we've saved the result into the dictionary we
# can add extra information we would like to return to
# the client.
#
request_body['target_bytes'] = result['target_bytes']
return request_body
def collect_legacy(self, stream, request_body, request):
''' Invoked when we must save the result of a session '''
ident = str(hash(stream)).encode('hex')
if ident not in self.clients:
#
# Before Neubot 0.4.2 we were using multiple connections
# for speedtest, which were used both for testing and for
# negotiating/collecting. Sometimes the connection used
# to collect is not the one used to negotiate: the code
# uses the one that terminates the upload first.
# When this happens we inspect the Authorization header
# before deciding the collect request is an abuse.
#
authorization = request['Authorization']
if authorization not in self.clients:
raise RuntimeError('Not authorized to collect')
else:
logging.warning('skype: working around multiple conns '
'issue')
ident = authorization
# Note: no more than one collect per session
self.clients.remove(ident)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
if 'privacy_can_share' in request_body:
request_body['privacy_can_publish'] = request_body[
'privacy_can_share']
del request_body['privacy_can_share']
if privacy.collect_allowed(request_body):
# TODO (claudiu) Implement BACKEND.skype_test_store()
BACKEND.speedtest_store(request_body)
else:
logging.warning('* bad privacy settings: %s', str(stream))
return {}
def test_failure(self):
''' Make sure collect_allowed() returns False on bad input '''
self.assertFalse(
privacy.collect_allowed({
'privacy_informed': 0,
'privacy_can_collect': 0,
'privacy_can_publish': 0
}))
self.assertFalse(
privacy.collect_allowed({
'privacy_informed': 0,
'privacy_can_collect': 0,
'privacy_can_publish': 1
}))
self.assertFalse(
privacy.collect_allowed({
'privacy_informed': 0,
'privacy_can_collect': 1,
'privacy_can_publish': 0
}))
self.assertFalse(
privacy.collect_allowed({
'privacy_informed': 0,
'privacy_can_collect': 1,
'privacy_can_publish': 1
}))
self.assertFalse(
privacy.collect_allowed({
'privacy_informed': 1,
'privacy_can_collect': 0,
'privacy_can_publish': 0
}))
self.assertFalse(
privacy.collect_allowed({
'privacy_informed': 1,
'privacy_can_collect': 0,
'privacy_can_publish': 1
}))
def process_request(self, stream, request):
""" Process rendezvous request """
if request["content-type"] == "application/json":
ibody = marshal.unmarshal_object(request.body.read(), "application/json", compat.RendezvousRequest)
else:
ibody = marshal.unmarshal_object(request.body.read(), "application/xml", compat.RendezvousRequest)
obody = compat.RendezvousResponse()
#
# If we don't say anything the rendezvous server is not
# going to prompt for updates. We need to specify the
# updated version number explicitly when we start it up.
# This should guarantee that we do not advertise -rc
# releases and other weird things.
#
version = self.conf["rendezvous.server.update_version"]
if version and ibody.version:
diff = utils_version.compare(version, ibody.version)
logging.debug("rendezvous: version=%s ibody.version=%s diff=%f", version, ibody.version, diff)
if diff > 0:
obody.update["uri"] = "http://neubot.org/"
obody.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# We only redirect to other servers clients that have
# agreed to give us the permission to publish, in order
# to be compliant with M-Lab policy.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default", "master.neubot.org")
logging.debug("* default test server: %s", server)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
request_body = ibody.__dict__.copy()
if "privacy_can_share" in request_body:
request_body["privacy_can_publish"] = request_body["privacy_can_share"]
del request_body["privacy_can_share"]
# Redirect IFF have ALL privacy permissions
if privacy.count_valid(request_body, "privacy_") == 3:
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(), country)
if not servers:
logging.info("* learning new country: %s", country)
table_geoloc.insert_server(DATABASE.connection(), country, server)
servers = [server]
server = random.choice(servers)
logging.info("rendezvous_server: %s[%s] -> %s", agent_address, country, server)
else:
logging.warning("rendezvous_server: cannot redirect to M-Lab: %s", request_body)
#
# We require at least informed and can_collect since 0.4.4
# (released 25 October 2011), so stop clients with empty
# privacy settings, who were still using master.
#
if privacy.collect_allowed(request_body):
#
# Note: Here we will have problems if we store unquoted
# IPv6 addresses into the database. Because the resulting
# URI won't be valid.
#
if "speedtest" in ibody.accept:
obody.available["speedtest"] = ["http://%s/speedtest" % server]
if "bittorrent" in ibody.accept:
obody.available["bittorrent"] = ["http://%s/" % server]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if ibody.version and utils_version.compare(ibody.version, "0.3.7") >= 0:
body = marshal.marshal_object(obody, "application/json")
mimetype = "application/json"
else:
body = compat.adhoc_marshaller(obody)
mimetype = "text/xml"
response = Message()
response.compose(code="200", reason="Ok", mimetype=mimetype, body=body)
stream.send_response(request, response)