def test_success(self): ''' Make sure collect_allowed() returns True on valid input ''' self.assertTrue(privacy.collect_allowed( {'privacy_informed': 1, 'privacy_can_collect': 1, 'privacy_can_publish': 0})) self.assertTrue(privacy.collect_allowed( {'privacy_informed': 1, 'privacy_can_collect': 1, 'privacy_can_publish': 1}))
def test_success(self): ''' Make sure collect_allowed() returns True on valid input ''' self.assertTrue( privacy.collect_allowed({ 'privacy_informed': 1, 'privacy_can_collect': 1, 'privacy_can_publish': 0 })) self.assertTrue( privacy.collect_allowed({ 'privacy_informed': 1, 'privacy_can_collect': 1, 'privacy_can_publish': 1 }))
def collect(self, m): btid = _make_btid(m["ident"]) if btid not in AUTH_PEERS: raise NegotiatorEOF() d = m["request_body"] result = AUTH_PEERS[btid] # # Note that the following is not a bug: it's just that # the server saves results using the point of view of the # client, i.e. upload_speed _is_ client's upload speed. # d["timestamp"] = result["timestamp"] d["upload_speed"] = result["upload_speed"] if privacy.collect_allowed(d): table_bittorrent.insert(DATABASE.connection(), d) # # After we've saved the result into the dictionary we # can add extra information we would like to return to # the client. # d["target_bytes"] = result["target_bytes"] m["response_body"] = d
def got_response_collecting(self, stream, request, response): LOG.complete() if self.success: # # Always measure at the receiver because there is more # information at the receiver and also to make my friend # Enrico happier :-P. # The following is not a bug: it's just that the server # returns a result using the point of view of the client, # i.e. upload_speed is _our_ upload speed. # m = json.loads(response.body.read()) self.my_side["upload_speed"] = m["upload_speed"] upload = utils.speed_formatter(m["upload_speed"]) STATE.update("test_upload", upload) if privacy.collect_allowed(self.my_side): table_bittorrent.insert(DATABASE.connection(), self.my_side) # Update the upstream channel estimate target_bytes = int(m["target_bytes"]) if target_bytes > 0: estimate.UPLOAD = target_bytes stream.close()
def got_response_collecting(self, stream, request, response): logging.info("BitTorrent: collecting ... done") if self.success: # # Always measure at the receiver because there is more # information at the receiver and also to make my friend # Enrico happier :-P. # The following is not a bug: it's just that the server # returns a result using the point of view of the client, # i.e. upload_speed is _our_ upload speed. # m = json.loads(response.body.read()) self.my_side["upload_speed"] = m["upload_speed"] upload = utils.speed_formatter(m["upload_speed"]) STATE.update("test_progress", "100%", publish=False) STATE.update("test_upload", upload) logging.info('BitTorrent: upload speed: %s', upload) if privacy.collect_allowed(self.my_side): if DATABASE.readonly: logging.warning('bittorrent_client: readonly database') else: table_bittorrent.insert(DATABASE.connection(), self.my_side) # Update the upstream channel estimate target_bytes = int(m["target_bytes"]) if target_bytes > 0: estimate.UPLOAD = target_bytes self.final_state = True stream.close()
def do_collect(self, stream, request): self._speedtest_complete(request) s = request.body.read() m = marshal.unmarshal_object(s, "text/xml", compat.SpeedtestCollect) if privacy.collect_allowed(m): table_speedtest.insertxxx(DATABASE.connection(), m) response = Message() response.compose(code="200", reason="Ok") stream.send_response(request, response)
def test_failure(self): ''' Make sure collect_allowed() returns False on bad input ''' self.assertFalse(privacy.collect_allowed( {'privacy_informed': 0, 'privacy_can_collect': 0, 'privacy_can_publish': 0})) self.assertFalse(privacy.collect_allowed( {'privacy_informed': 0, 'privacy_can_collect': 0, 'privacy_can_publish': 1})) self.assertFalse(privacy.collect_allowed( {'privacy_informed': 0, 'privacy_can_collect': 1, 'privacy_can_publish': 0})) self.assertFalse(privacy.collect_allowed( {'privacy_informed': 0, 'privacy_can_collect': 1, 'privacy_can_publish': 1})) self.assertFalse(privacy.collect_allowed( {'privacy_informed': 1, 'privacy_can_collect': 0, 'privacy_can_publish': 0})) self.assertFalse(privacy.collect_allowed( {'privacy_informed': 1, 'privacy_can_collect': 0, 'privacy_can_publish': 1}))
def connection_ready(self, stream): m1 = SpeedtestCollect() m1.client = self.conf.get("uuid", "") m1.timestamp = utils.timestamp() m1.internalAddress = stream.myname[0] m1.realAddress = self.conf.get("speedtest.client.public_address", "") m1.remoteAddress = stream.peername[0] m1.latency = self.conf.get("speedtest.client.latency", 0.0) m1.downloadSpeed = self.conf.get("speedtest.client.download", 0.0) m1.uploadSpeed = self.conf.get("speedtest.client.upload", 0.0) m1.privacy_informed = self.conf.get("privacy.informed", 0) m1.privacy_can_collect = self.conf.get("privacy.can_collect", 0) m1.privacy_can_share = self.conf.get("privacy.can_publish", 0) # XXX m1.neubot_version = utils_version.NUMERIC_VERSION m1.platform = sys.platform m1.connectTime = sum(self.rtts) / len(self.rtts) # Test version (added Neubot 0.4.12) m1.testVersion = CONFIG["speedtest_test_version"] s = marshal.marshal_object(m1, "text/xml") stringio = StringIO.StringIO(s) # # Pass a dictionary because the function does not accept # anymore an object # if privacy.collect_allowed(m1.__dict__): if DATABASE.readonly: logging.warning("speedtest: readonly database") else: insertxxx(DATABASE.connection(), m1) request = Message() request.compose( method="POST", pathquery="/speedtest/collect", body=stringio, mimetype="application/xml", host=self.host_header, ) request["authorization"] = self.conf.get("speedtest.client.authorization", "") stream.send_request(request)
def connection_ready(self, stream): m1 = compat.SpeedtestCollect() m1.client = self.conf.get("uuid", "") m1.timestamp = utils.timestamp() m1.internalAddress = stream.myname[0] m1.realAddress = self.conf.get("speedtest.client.public_address", "") m1.remoteAddress = stream.peername[0] m1.latency = self.conf.get("speedtest.client.latency", 0.0) m1.downloadSpeed = self.conf.get("speedtest.client.download", 0.0) m1.uploadSpeed = self.conf.get("speedtest.client.upload", 0.0) m1.privacy_informed = self.conf.get("privacy.informed", 0) m1.privacy_can_collect = self.conf.get("privacy.can_collect", 0) m1.privacy_can_share = self.conf.get("privacy.can_share", 0) m1.neubot_version = LibVersion.to_numeric("0.4.2") m1.platform = sys.platform if self.measurer: m1.connectTime = self.measurer.measure_rtt()[0] # import pprint # if hasattr(self.measurer, "recv_hist"): # download = self.measurer.recv_hist.get("download", []) # pprint.pprint(download) # if hasattr(self.measurer, "send_hist"): # upload = self.measurer.send_hist.get("upload", []) # pprint.pprint(upload) s = marshal.marshal_object(m1, "text/xml") stringio = StringIO.StringIO(s) if privacy.collect_allowed(m1): table_speedtest.insertxxx(DATABASE.connection(), m1) request = Message() request.compose(method="POST", pathquery="/speedtest/collect", body=stringio, mimetype="application/xml", host=self.host_header) request["authorization"] = self.conf.get( "speedtest.client.authorization", "") stream.send_request(request)
def collect(self, stream, request_body): ''' Invoked when we must save the result of a session ''' sha1 = self._stream_to_sha1(stream) if sha1 not in self.peers: raise RuntimeError('Not authorized to collect') else: # Note: no more than one collect per session result = self.peers[sha1] del self.peers[sha1] # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # if 'privacy_can_share' in request_body: request_body['privacy_can_publish'] = request_body[ 'privacy_can_share'] del request_body['privacy_can_share'] # # Note that the following is not a bug: it's just that # the server saves results using the point of view of the # client, i.e. upload_speed _is_ client's upload speed. # request_body['timestamp'] = result['timestamp'] request_body['upload_speed'] = result['upload_speed'] if privacy.collect_allowed(request_body): BACKEND.bittorrent_store(request_body) else: logging.warning('* bad privacy settings: %s', str(stream)) # # After we've saved the result into the dictionary we # can add extra information we would like to return to # the client. # request_body['target_bytes'] = result['target_bytes'] return request_body
def collect_legacy(self, stream, request_body, request): ''' Invoked when we must save the result of a session ''' ident = str(hash(stream)).encode('hex') if ident not in self.clients: # # Before Neubot 0.4.2 we were using multiple connections # for speedtest, which were used both for testing and for # negotiating/collecting. Sometimes the connection used # to collect is not the one used to negotiate: the code # uses the one that terminates the upload first. # When this happens we inspect the Authorization header # before deciding the collect request is an abuse. # authorization = request['Authorization'] if authorization not in self.clients: raise RuntimeError('Not authorized to collect') else: logging.warning('skype: working around multiple conns ' 'issue') ident = authorization # Note: no more than one collect per session self.clients.remove(ident) # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # if 'privacy_can_share' in request_body: request_body['privacy_can_publish'] = request_body[ 'privacy_can_share'] del request_body['privacy_can_share'] if privacy.collect_allowed(request_body): # TODO (claudiu) Implement BACKEND.skype_test_store() BACKEND.speedtest_store(request_body) else: logging.warning('* bad privacy settings: %s', str(stream)) return {}
def test_failure(self): ''' Make sure collect_allowed() returns False on bad input ''' self.assertFalse( privacy.collect_allowed({ 'privacy_informed': 0, 'privacy_can_collect': 0, 'privacy_can_publish': 0 })) self.assertFalse( privacy.collect_allowed({ 'privacy_informed': 0, 'privacy_can_collect': 0, 'privacy_can_publish': 1 })) self.assertFalse( privacy.collect_allowed({ 'privacy_informed': 0, 'privacy_can_collect': 1, 'privacy_can_publish': 0 })) self.assertFalse( privacy.collect_allowed({ 'privacy_informed': 0, 'privacy_can_collect': 1, 'privacy_can_publish': 1 })) self.assertFalse( privacy.collect_allowed({ 'privacy_informed': 1, 'privacy_can_collect': 0, 'privacy_can_publish': 0 })) self.assertFalse( privacy.collect_allowed({ 'privacy_informed': 1, 'privacy_can_collect': 0, 'privacy_can_publish': 1 }))
def process_request(self, stream, request): """ Process rendezvous request """ if request["content-type"] == "application/json": ibody = marshal.unmarshal_object(request.body.read(), "application/json", compat.RendezvousRequest) else: ibody = marshal.unmarshal_object(request.body.read(), "application/xml", compat.RendezvousRequest) obody = compat.RendezvousResponse() # # If we don't say anything the rendezvous server is not # going to prompt for updates. We need to specify the # updated version number explicitly when we start it up. # This should guarantee that we do not advertise -rc # releases and other weird things. # version = self.conf["rendezvous.server.update_version"] if version and ibody.version: diff = utils_version.compare(version, ibody.version) logging.debug("rendezvous: version=%s ibody.version=%s diff=%f", version, ibody.version, diff) if diff > 0: obody.update["uri"] = "http://neubot.org/" obody.update["version"] = version # # Select test server address. # The default test server is the master server itself. # If we know the country, lookup the list of servers for # that country in the database. # We only redirect to other servers clients that have # agreed to give us the permission to publish, in order # to be compliant with M-Lab policy. # If there are no servers for that country, register # the master server for the country so that we can notice # we have new users and can take the proper steps to # deploy nearby servers. # server = self.conf.get("rendezvous.server.default", "master.neubot.org") logging.debug("* default test server: %s", server) # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # request_body = ibody.__dict__.copy() if "privacy_can_share" in request_body: request_body["privacy_can_publish"] = request_body["privacy_can_share"] del request_body["privacy_can_share"] # Redirect IFF have ALL privacy permissions if privacy.count_valid(request_body, "privacy_") == 3: agent_address = stream.peername[0] country = GEOLOCATOR.lookup_country(agent_address) if country: servers = table_geoloc.lookup_servers(DATABASE.connection(), country) if not servers: logging.info("* learning new country: %s", country) table_geoloc.insert_server(DATABASE.connection(), country, server) servers = [server] server = random.choice(servers) logging.info("rendezvous_server: %s[%s] -> %s", agent_address, country, server) else: logging.warning("rendezvous_server: cannot redirect to M-Lab: %s", request_body) # # We require at least informed and can_collect since 0.4.4 # (released 25 October 2011), so stop clients with empty # privacy settings, who were still using master. # if privacy.collect_allowed(request_body): # # Note: Here we will have problems if we store unquoted # IPv6 addresses into the database. Because the resulting # URI won't be valid. # if "speedtest" in ibody.accept: obody.available["speedtest"] = ["http://%s/speedtest" % server] if "bittorrent" in ibody.accept: obody.available["bittorrent"] = ["http://%s/" % server] # # Neubot <=0.3.7 expects to receive an XML document while # newer Neubots want a JSON. I hope old clients will upgrade # pretty soon. # if ibody.version and utils_version.compare(ibody.version, "0.3.7") >= 0: body = marshal.marshal_object(obody, "application/json") mimetype = "application/json" else: body = compat.adhoc_marshaller(obody) mimetype = "text/xml" response = Message() response.compose(code="200", reason="Ok", mimetype=mimetype, body=body) stream.send_response(request, response)