def test_good_cases(self):
''' Make sure it correctly counts the number of valid settings '''
for prefix in ('privacy.', 'privacy_'):
for trueval in (1, '1', 'true', 'on', 'yes'):
self.assertEqual(
1,
privacy.count_valid({'%sinformed' % prefix: trueval},
prefix))
self.assertEqual(
1,
privacy.count_valid({'%scan_collect' % prefix: trueval},
prefix))
self.assertEqual(
1,
privacy.count_valid({'%scan_publish' % prefix: trueval},
prefix))
self.assertEqual(
3,
privacy.count_valid(
{
'%sinformed' % prefix: trueval,
'%scan_collect' % prefix: trueval,
'%scan_publish' % prefix: trueval
}, prefix))
def test_empty_cases(self):
''' Make sure it returns 0 when there is nothing interesting '''
# Nothing in input
self.assertEqual(0, privacy.count_valid({}, ""))
# Wrong prefix
self.assertEqual(
0, privacy.count_valid({'privacy.informed': 1}, "privacy_"))
def test_empty_cases(self):
''' Make sure it returns 0 when there is nothing interesting '''
# Nothing in input
self.assertEqual(0, privacy.count_valid(
{}, ""))
# Wrong prefix
self.assertEqual(0, privacy.count_valid(
{'privacy.informed': 1}, "privacy_"))
def test_bad(self):
''' Make sure it returns -1 when settings are bad '''
# False value
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 0}, "privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.can_collect': 0}, "privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.can_publish': 0}, "privacy."))
# Not all are True
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 0,
'privacy.can_collect': 0,
'privacy.can_publish': 0},
"privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 0,
'privacy.can_collect': 0,
'privacy.can_publish': 1},
"privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 0,
'privacy.can_collect': 1,
'privacy.can_publish': 0},
"privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 0,
'privacy.can_collect': 1,
'privacy.can_publish': 1},
"privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 1,
'privacy.can_collect': 0,
'privacy.can_publish': 0},
"privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 1,
'privacy.can_collect': 0,
'privacy.can_publish': 1},
"privacy."))
self.assertEqual(-1, privacy.count_valid(
{'privacy.informed': 1,
'privacy.can_collect': 1,
'privacy.can_publish': 0},
"privacy."))
def _do_run_queue(self, first_elem):
''' Actually run first element in queue '''
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
raise RuntimeError('runner_core: bad privacy settings')
elif first_elem[0] == 'rendezvous':
runner_rendezvous.run(conf['agent.master'], '9773')
elif first_elem[0] == 'speedtest':
uri = RUNNER_TESTS.test_to_negotiate_uri('speedtest')
conf['speedtest.client.uri'] = uri
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
elif first_elem[0] == 'bittorrent':
uri = RUNNER_TESTS.test_to_negotiate_uri('bittorrent')
conf['bittorrent._uri'] = uri
bittorrent.run(POLLER, conf)
elif first_elem[0] == 'dload':
RunnerDload(first_elem[2])
elif first_elem[0] == 'raw':
address = RUNNER_HOSTS.get_random_host()
handler = RawNegotiate()
handler.connect((address, 8080), CONFIG['prefer_ipv6'], 0, {})
elif first_elem[0] == 'mlab-ns':
handler = RunnerMlabns()
if not first_elem[2]:
extra = {'policy': ''} # get closest server by default
else:
extra = first_elem[2]
handler.connect(('mlab-ns.appspot.com', 80),
CONFIG['prefer_ipv6'], 0, extra)
elif first_elem[0] in self.dynamic_tests:
address = RUNNER_HOSTS.get_random_host()
port = 80 # XXX
self.dynamic_tests[first_elem[0]]["test_func"]({
"address": address,
"conf": CONFIG.copy(),
"poller": POLLER,
"port": port,
})
else:
raise RuntimeError('runner_core: asked to run an unknown test')
def test_good_cases(self):
''' Make sure it correctly counts the number of valid settings '''
for prefix in ('privacy.', 'privacy_'):
for trueval in (1, '1', 'true', 'on', 'yes'):
self.assertEqual(1, privacy.count_valid(
{'%sinformed' % prefix: trueval},
prefix))
self.assertEqual(1, privacy.count_valid(
{'%scan_collect' % prefix: trueval},
prefix))
self.assertEqual(1, privacy.count_valid(
{'%scan_publish' % prefix: trueval},
prefix))
self.assertEqual(3, privacy.count_valid(
{'%sinformed' % prefix: trueval,
'%scan_collect' % prefix: trueval,
'%scan_publish' % prefix: trueval},
prefix))
def _do_run_queue(self):
''' Actually run first element in queue '''
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
raise RuntimeError('Bad privacy settings')
# Run rendezvous
elif self.queue[0][0] == 'rendezvous':
uri = "http://%s:9773/rendezvous" % conf['agent.master']
runner_rendezvous.run(uri)
# Run speedtest
elif self.queue[0][0] == 'speedtest':
uri = RUNNER_TESTS.test_to_negotiate_uri('speedtest')
#
# If we have no negotiate URI for this test, possibly
# because we are offline, abort it.
#
if not uri:
raise RuntimeError('No negotiate URI for speedtest')
conf['speedtest.client.uri'] = uri
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
# Run bittorrent
elif self.queue[0][0] == 'bittorrent':
uri = RUNNER_TESTS.test_to_negotiate_uri('bittorrent')
#
# If we have no negotiate URI for this test, possibly
# because we are offline, abort it.
#
if not uri:
raise RuntimeError('No negotiate URI for bittorrent')
conf['bittorrent._uri'] = uri
bittorrent.run(POLLER, conf)
# Run dload
elif self.queue[0][0] == 'dload':
RunnerDload(self.queue[0][2])
# Safety net
else:
raise RuntimeError('Asked to run an unknown test')
def __should_adjust_privacy(database_path):
''' Connect to the daemon, get privacy settings and return
true if the user should adjust privacy settings '''
#
# Portions of this function can be shared between this
# notifier and the UNIX one. At the moment there's a
# small amount of code duplication.
#
try:
address, port = '127.0.0.1', '9774'
connection = sqlite3.connect(database_path)
cursor = connection.cursor()
cursor.execute('SELECT * FROM config;')
for name, value in cursor:
if name == 'agent.api.address':
address = value
elif name == 'agent.api.port':
port = value
connection.close()
connection = httplib.HTTPConnection(address, port)
connection.request('GET', '/api/config')
response = connection.getresponse()
if response.status != 200:
raise RuntimeError('Invalid response code: %d' % response.status)
body = response.read()
connection.close()
dictionary = json.loads(body)
if privacy.count_valid(dictionary, 'privacy.') != 3:
# Should adjust settings
return "http://%s:%s/" % (address, port)
except SystemExit:
raise
except:
syslog.syslog(syslog.LOG_ERR, '%s' %
str(asyncore.compact_traceback()))
# No need to adjust settings
return None
def run_queue(self):
''' If possible run the first test in queue '''
# Adapted from neubot/rendezvous/client.py
if not self.queue:
return
if self.running:
return
#
# Subscribe BEFORE starting the test, otherwise we
# may miss the 'testdone' event if the connection
# to the negotiator service fails, and we will stay
# stuck forever.
#
NOTIFIER.subscribe('testdone', self.test_done)
# Prevent concurrent tests
self.running = True
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
NOTIFIER.publish('testdone')
# Run speedtest
elif self.queue[0][0] == 'speedtest':
conf['speedtest.client.uri'] = self.queue[0][1]
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
# Run bittorrent
elif self.queue[0][0] == 'bittorrent':
conf['bittorrent._uri'] = self.queue[0][1]
bittorrent.run(POLLER, conf)
# Safety net
else:
LOG.error('Asked to run an unknown test')
NOTIFIER.publish('testdone')
def __should_adjust_privacy(database_path):
''' Connect to the daemon, get privacy settings and return
true if the user should adjust privacy settings '''
try:
address, port = '127.0.0.1', '9774'
connection = sqlite3.connect(database_path)
cursor = connection.cursor()
cursor.execute('SELECT * FROM config;')
for name, value in cursor:
if name == 'agent.api.address':
address = value
elif name == 'agent.api.port':
port = value
connection.close()
connection = lib_http.HTTPConnection(address, port)
connection.request('GET', '/api/config')
response = connection.getresponse()
if response.status != 200:
raise RuntimeError('Invalid response code: %d' % response.status)
body = response.read()
connection.close()
dictionary = json.loads(body)
if privacy.count_valid(dictionary, 'privacy.') != 3:
# Should adjust settings
return True
except SystemExit:
raise
except:
syslog.syslog(syslog.LOG_ERR, '%s' %
str(asyncore.compact_traceback()))
# No need to adjust settings
return False
def process_request(self, stream, request):
m = marshal.unmarshal_object(request.body.read(),
"application/xml", compat.RendezvousRequest)
m1 = compat.RendezvousResponse()
#
# If we don't say anything the rendezvous server is not
# going to prompt for updates. We need to specify the
# updated version number explicitly when we start it up.
# This should guarantee that we do not advertise -rc
# releases and other weird things.
#
version = self.conf["rendezvous.server.update_version"]
if version and m.version:
diff = LibVersion.compare(version, m.version)
LOG.debug('rendezvous: version=%s m.version=%s diff=%f' % (
version, m.version, diff))
if diff > 0:
m1.update["uri"] = self.conf["rendezvous.server.update_uri"]
m1.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# We only redirect to other servers clients that have
# agreed to give us the permission to publish, in order
# to be compliant with M-Lab policy.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default",
"master.neubot.org")
LOG.debug("* default test server: %s" % server)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
request_body = m.__dict__.copy()
if 'privacy_can_share' in request_body:
request_body['privacy_can_publish'] = request_body[
'privacy_can_share']
del request_body['privacy_can_share']
# Redirect IFF have ALL privacy permissions
if privacy.count_valid(request_body, 'privacy_') == 3:
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(),
country)
if not servers:
LOG.info("* learning new country: %s" % country)
table_geoloc.insert_server(DATABASE.connection(),
country, server)
servers = [server]
server = random.choice(servers)
LOG.debug("* selected test server: %s" % server)
if "speedtest" in m.accept:
m1.available["speedtest"] = [ "http://%s/speedtest" % server ]
if "bittorrent" in m.accept:
m1.available["bittorrent"] = [ "http://%s/" % server ]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if m.version and LibVersion.compare(m.version, "0.3.7") >= 0:
s = marshal.marshal_object(m1, "application/json")
mimetype = "application/json"
else:
s = compat.adhoc_marshaller(m1)
mimetype = "text/xml"
stringio = StringIO.StringIO()
stringio.write(s)
stringio.seek(0)
response = Message()
response.compose(code="200", reason="Ok",
mimetype=mimetype, body=stringio)
stream.send_response(request, response)
def config_api(stream, request, query):
''' Implements /api/config API '''
# Adapted from neubot/api/server.py
#
# Fetch and process common options from the query
# string, for now the only implemented option is
# debug, which modifies the semantic to return text
# for humans instead of JSON.
#
mimetype = 'application/json'
indent = None
options = cgi.parse_qs(query)
if utils.intify(options.get('debug', ['0'])[0]):
mimetype = 'text/plain'
indent = 4
#
# Now that we know the response format, decide what is
# the content of the response. If the labels option is
# available we return the documentation coupled with a
# setting. When the method is not POST, return instead
# the name and value of each setting.
#
if utils.intify(options.get('labels', ['0'])[0]):
obj = CONFIG.descriptions
elif request.method != 'POST':
obj = CONFIG.conf
else:
#
# When the method is POST we need to read the
# new settings from the request body. Settings
# are a x-www-form-urlencoded dictionary to
# ease AJAX programming.
#
body = request.body.read()
updates = marshal.qs_to_dictionary(body)
#
# PRE-update checks. We need to make sure that
# the following things are True:
#
# 1. that the incoming dictionary does not contain
# invalid privacy settings;
#
# 2. that the interval between automatic tests is
# either reasonable or set to zero, which means
# that it needs to be extracted randomly.
#
count = privacy.count_valid(updates, 'privacy.')
if count < 0:
raise ConfigError('Passed invalid privacy settings')
agent_interval = int(updates.get('agent.interval', 0))
if agent_interval != 0 and agent_interval < 1380:
raise ConfigError('Passed invalid agent.interval')
# Merge settings
CONFIG.merge_api(updates, DATABASE.connection())
#
# Update the state, such that, if the AJAX code is
# tracking the state it gets a notification that
# some configurations variable have been modified.
# Given that we communicate the update via that
# channel, the response body is an empty dict to
# keep happy the AJAX code.
#
STATE.update('config', updates)
obj = '{}'
#
# Now that we know the body, prepare and send
# the response for the client.
#
response = Message()
body = json.dumps(obj, sort_keys=True, indent=indent)
response.compose(code="200", reason="Ok", body=body, mimetype=mimetype)
stream.send_response(request, response)
def process_request(self, stream, request):
""" Process rendezvous request """
if request["content-type"] == "application/json":
ibody = marshal.unmarshal_object(request.body.read(), "application/json", compat.RendezvousRequest)
else:
ibody = marshal.unmarshal_object(request.body.read(), "application/xml", compat.RendezvousRequest)
obody = compat.RendezvousResponse()
#
# If we don't say anything the rendezvous server is not
# going to prompt for updates. We need to specify the
# updated version number explicitly when we start it up.
# This should guarantee that we do not advertise -rc
# releases and other weird things.
#
version = self.conf["rendezvous.server.update_version"]
if version and ibody.version:
diff = utils_version.compare(version, ibody.version)
logging.debug("rendezvous: version=%s ibody.version=%s diff=%f", version, ibody.version, diff)
if diff > 0:
obody.update["uri"] = "http://neubot.org/"
obody.update["version"] = version
#
# Select test server address.
# The default test server is the master server itself.
# If we know the country, lookup the list of servers for
# that country in the database.
# We only redirect to other servers clients that have
# agreed to give us the permission to publish, in order
# to be compliant with M-Lab policy.
# If there are no servers for that country, register
# the master server for the country so that we can notice
# we have new users and can take the proper steps to
# deploy nearby servers.
#
server = self.conf.get("rendezvous.server.default", "master.neubot.org")
logging.debug("* default test server: %s", server)
#
# Backward compatibility: the variable name changed from
# can_share to can_publish after Neubot 0.4.5
#
request_body = ibody.__dict__.copy()
if "privacy_can_share" in request_body:
request_body["privacy_can_publish"] = request_body["privacy_can_share"]
del request_body["privacy_can_share"]
# Redirect IFF have ALL privacy permissions
if privacy.count_valid(request_body, "privacy_") == 3:
agent_address = stream.peername[0]
country = GEOLOCATOR.lookup_country(agent_address)
if country:
servers = table_geoloc.lookup_servers(DATABASE.connection(), country)
if not servers:
logging.info("* learning new country: %s", country)
table_geoloc.insert_server(DATABASE.connection(), country, server)
servers = [server]
server = random.choice(servers)
logging.info("rendezvous_server: %s[%s] -> %s", agent_address, country, server)
else:
logging.warning("rendezvous_server: cannot redirect to M-Lab: %s", request_body)
#
# We require at least informed and can_collect since 0.4.4
# (released 25 October 2011), so stop clients with empty
# privacy settings, who were still using master.
#
if privacy.collect_allowed(request_body):
#
# Note: Here we will have problems if we store unquoted
# IPv6 addresses into the database. Because the resulting
# URI won't be valid.
#
if "speedtest" in ibody.accept:
obody.available["speedtest"] = ["http://%s/speedtest" % server]
if "bittorrent" in ibody.accept:
obody.available["bittorrent"] = ["http://%s/" % server]
#
# Neubot <=0.3.7 expects to receive an XML document while
# newer Neubots want a JSON. I hope old clients will upgrade
# pretty soon.
#
if ibody.version and utils_version.compare(ibody.version, "0.3.7") >= 0:
body = marshal.marshal_object(obody, "application/json")
mimetype = "application/json"
else:
body = compat.adhoc_marshaller(obody)
mimetype = "text/xml"
response = Message()
response.compose(code="200", reason="Ok", mimetype=mimetype, body=body)
stream.send_response(request, response)
def test_bad(self):
''' Make sure it returns -1 when settings are bad '''
# False value
self.assertEqual(
-1, privacy.count_valid({'privacy.informed': 0}, "privacy."))
self.assertEqual(
-1, privacy.count_valid({'privacy.can_collect': 0}, "privacy."))
self.assertEqual(
-1, privacy.count_valid({'privacy.can_publish': 0}, "privacy."))
# Not all are True
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 0,
'privacy.can_collect': 0,
'privacy.can_publish': 0
}, "privacy."))
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 0,
'privacy.can_collect': 0,
'privacy.can_publish': 1
}, "privacy."))
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 0,
'privacy.can_collect': 1,
'privacy.can_publish': 0
}, "privacy."))
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 0,
'privacy.can_collect': 1,
'privacy.can_publish': 1
}, "privacy."))
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 1,
'privacy.can_collect': 0,
'privacy.can_publish': 0
}, "privacy."))
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 1,
'privacy.can_collect': 0,
'privacy.can_publish': 1
}, "privacy."))
self.assertEqual(
-1,
privacy.count_valid(
{
'privacy.informed': 1,
'privacy.can_collect': 1,
'privacy.can_publish': 0
}, "privacy."))
