def _loop_once(args):
message = ""
# Check for updates
try:
connection = httplib.HTTPConnection("127.0.0.1", "9774")
connection.request("GET", "/api/state")
response = connection.getresponse()
if response.status != 200:
raise RuntimeError("Unexpected response")
body = response.read()
dictionary = json.loads(body)
update = dictionary["events"]["update"]
tpl = update["version"], update["uri"]
message += "New version %s available at <%s> " % tpl
except:
LOG.exception()
# Check whether we need to update privacy settings
try:
connection = httplib.HTTPConnection("127.0.0.1", "9774")
connection.request("GET", "/api/config")
response = connection.getresponse()
if response.status != 200:
raise RuntimeError("Unexpected response")
body = response.read()
dictionary = json.loads(body)
if (not "privacy.informed" in dictionary or not
dictionary["privacy.informed"]):
uri = "http://127.0.0.1:9774/privacy.html"
message += " Please update your privacy settings at <%s>" % uri
# TODO Does the Law allow to force that at the /api level?
if ("privacy.informed" in dictionary and
"privacy.can_collect" in dictionary and
dictionary["privacy.informed"] and
not dictionary["privacy.can_collect"]):
uri = "http://127.0.0.1:9774/privacy.html"
message += " How is Neubot supposed to work if it cannot\n" \
"save the results of your tests? Please update\n" \
"your privacy settings at <%s>" % uri
except:
LOG.exception()
# Spam the user
if message:
InfoBox(message, 300)
def check_response(self, response):
if response.code != "200":
raise ValueError("Bad HTTP response code")
if response["content-type"] != "application/json":
raise ValueError("Unexpected contenty type")
octets = response.body.read()
dictionary = json.loads(octets)
LOG.debug("APIStateTracker: received JSON: " +
json.dumps(dictionary, ensure_ascii=True))
if not "events" in dictionary:
return
if not "current" in dictionary:
raise ValueError("Incomplete dictionary")
t = dictionary["t"]
if not type(t) == types.IntType and not type(t) == types.LongType:
raise ValueError("Invalid type for current event time")
if t < 0:
raise ValueError("Invalid value for current event time")
self.timestamp = t
self.process_dictionary(dictionary)
def main(args):
''' main function '''
try:
options, arguments = getopt.getopt(args[1:], 'b:d:m:t:v')
except getopt.error:
sys.exit(USAGE)
if arguments:
sys.exit(USAGE)
# Good-enough default message
default_msg = {
"timestamp": 0,
"uuid": "",
"internal_address": "",
"real_address": "",
"remote_address": "",
"privacy_informed": 0,
"privacy_can_collect": 0,
"privacy_can_publish": 0,
"latency": 0.0,
"connect_time": 0.0,
"download_speed": 0.0,
"upload_speed": 0.0,
"neubot_version": "",
"platform": "",
}
bcknd = None
datadir = None
msg = default_msg
timestamp = None
for name, value in options:
if name == '-b':
bcknd = value
elif name == '-d':
datadir = value
if name == '-m':
msg = value
elif name == '-t':
timestamp = float(value)
elif name == '-v':
logging.getLogger().setLevel(logging.DEBUG)
if bcknd:
BACKEND.use_backend(bcknd)
if datadir:
FILESYS.datadir = datadir
if msg != default_msg:
msg = json.loads(msg)
if timestamp:
time.time = lambda: timestamp
FILESYS.datadir_init()
BACKEND.bittorrent_store(msg)
BACKEND.speedtest_store(msg)
def api_results(stream, request, query):
''' Populates www/results.html page '''
dictionary = cgi.parse_qs(query)
test = CONFIG['www_default_test_to_show']
if 'test' in dictionary:
test = str(dictionary['test'][0])
# Read the directory each time, so you don't need to restart the daemon
# after you have changed the description of a test.
available_tests = {}
for filename in os.listdir(TESTDIR):
if filename.endswith('.json'):
index = filename.rfind('.json')
if index == -1:
raise RuntimeError('api_results: internal error')
name = filename[:index]
available_tests[name] = filename
if not test in available_tests:
raise NotImplementedTest('Test not implemented')
# Allow power users to customize results.html heavily, by creating JSON
# descriptions with local modifications.
filepath = utils_path.append(TESTDIR, available_tests[test], False)
if not filepath:
raise RuntimeError("api_results: append() path failed")
localfilepath = filepath + '.local'
if os.path.isfile(localfilepath):
filep = open(localfilepath, 'rb')
else:
filep = open(filepath, 'rb')
response_body = json.loads(filep.read())
filep.close()
# Add extra information needed to populate results.html selection that
# allows to select which test results must be shown.
response_body['available_tests'] = available_tests.keys()
response_body['selected_test'] = test
descrpath = filepath.replace('.json', '.html')
if os.path.isfile(descrpath):
filep = open(descrpath, 'rb')
response_body['description'] = filep.read()
filep.close()
# Provide the web user interface some settings it needs, but only if they
# were not already provided by the `.local` file.
for variable in COPY_CONFIG_VARIABLES:
if not variable in response_body:
response_body[variable] = CONFIG[variable]
# Note: DO NOT sort keys here: order MUST be preserved
indent, mimetype = None, 'application/json'
if 'debug' in dictionary and utils.intify(dictionary['debug'][0]):
indent, mimetype = 4, 'text/plain'
response = Message()
body = json.dumps(response_body, indent=indent)
response.compose(code='200', reason='Ok', body=body, mimetype=mimetype)
stream.send_response(request, response)
def got_response_negotiating(self, stream, request, response):
m = json.loads(response.body.read())
PROPERTIES = ("authorization", "queue_pos", "real_address", "unchoked")
for k in PROPERTIES:
self.conf["_%s" % k] = m[k]
if not self.conf["_unchoked"]:
LOG.complete("done (queue_pos %d)" % m["queue_pos"])
STATE.update("negotiate", {"queue_pos": m["queue_pos"]})
self.connection_ready(stream)
else:
LOG.complete("done (unchoked)")
sha1 = hashlib.sha1()
sha1.update(m["authorization"])
self.conf["bittorrent.my_id"] = sha1.digest()
LOG.debug("* My ID: %s" % sha1.hexdigest())
self.http_stream = stream
self.negotiating = False
peer = PeerNeubot(self.poller)
peer.complete = self.peer_test_complete
peer.connection_lost = self.peer_connection_lost
peer.connection_failed = self.peer_connection_failed
peer.configure(self.conf)
peer.connect((self.http_stream.peername[0],
self.conf["bittorrent.port"]))
def got_response_collecting(self, stream, request, response):
logging.info("BitTorrent: collecting ... done")
if self.success:
#
# Always measure at the receiver because there is more
# information at the receiver and also to make my friend
# Enrico happier :-P.
# The following is not a bug: it's just that the server
# returns a result using the point of view of the client,
# i.e. upload_speed is _our_ upload speed.
#
m = json.loads(response.body.read())
self.my_side["upload_speed"] = m["upload_speed"]
upload = utils.speed_formatter(m["upload_speed"])
STATE.update("test_progress", "100%", publish=False)
STATE.update("test_upload", upload)
logging.info('BitTorrent: upload speed: %s', upload)
if privacy.collect_allowed(self.my_side):
if DATABASE.readonly:
logging.warning('bittorrent_client: readonly database')
else:
table_bittorrent.insert(DATABASE.connection(), self.my_side)
# Update the upstream channel estimate
target_bytes = int(m["target_bytes"])
if target_bytes > 0:
estimate.UPLOAD = target_bytes
self.final_state = True
stream.close()
def got_response_collecting(self, stream, request, response):
LOG.complete()
if self.success:
#
# Always measure at the receiver because there is more
# information at the receiver and also to make my friend
# Enrico happier :-P.
# The following is not a bug: it's just that the server
# returns a result using the point of view of the client,
# i.e. upload_speed is _our_ upload speed.
#
m = json.loads(response.body.read())
self.my_side["upload_speed"] = m["upload_speed"]
upload = utils.speed_formatter(m["upload_speed"])
STATE.update("test_upload", upload)
if privacy.collect_allowed(self.my_side):
table_bittorrent.insert(DATABASE.connection(), self.my_side)
# Update the upstream channel estimate
target_bytes = int(m["target_bytes"])
if target_bytes > 0:
estimate.UPLOAD = target_bytes
stream.close()
def main(args):
''' Monitor Neubot state via command line '''
try:
options, arguments = getopt.getopt(args[1:], 'D:v')
except getopt.error:
sys.exit('Usage: neubot api.client [-v] [-D property=value]')
if arguments:
sys.exit('Usage: neubot api.client [-v] [-D property=value]')
address, port, verbosity = '127.0.0.1', '9774', 0
for name, value in options:
if name == '-D':
name, value = value.split('=', 1)
if name == 'address':
address = value
elif name == 'port':
port = value
elif name == '-v':
verbosity += 1
timestamp = 0
while True:
try:
connection = lib_http.HTTPConnection(address, port)
connection.set_debuglevel(verbosity)
connection.request('GET', '/api/state?t=%d' % timestamp)
response = connection.getresponse()
if response.status != 200:
raise RuntimeError('Bad HTTP status: %d' % response.status)
if response.getheader("content-type") != "application/json":
raise RuntimeError("Unexpected contenty type")
octets = response.read()
dictionary = json.loads(octets)
logging.info("APIStateTracker: received JSON: %s",
json.dumps(dictionary, ensure_ascii=True))
if not "events" in dictionary:
continue
if not "current" in dictionary:
raise RuntimeError("Incomplete dictionary")
timestamp = max(0, int(dictionary["t"]))
json.dumps(dictionary, sys.stdout)
except KeyboardInterrupt:
break
except:
error = asyncore.compact_traceback()
logging.error('Exception: %s', str(error))
time.sleep(5)
def test_choke(self):
"""Verify finalize_response() when choke"""
dummy = [False]
module = NegotiatorModule()
module.unchoke = lambda m: dummy.pop()
negotiator = _Negotiator()
negotiator.register("abc", module)
m = { "response_body": {}, "module": "abc" }
negotiator._finalize_response(m, 21)
self.assertEqual(json.loads(m["response_body"]),
{ u"unchoked": False, u"queue_pos": 21 })
self.assertEqual(len(dummy), 1)
def handle_end_of_body(self, stream):
HttpClient.handle_end_of_body(self, stream)
context = stream.opaque
extra = context.extra
if extra['requests'] <= 0:
raise RuntimeError('runner_mlabns: unexpected response')
extra['requests'] -= 1
tmp = context.headers.get(CONTENT_TYPE)
if context.code != CODE200 or tmp != APPLICATION_JSON:
logging.error('runner_mlabns: bad response')
stream.close()
return
content = six.bytes_to_string(context.body.getvalue(), 'utf-8')
response = json.loads(content)
http_utils.prettyprint_json(response, '<')
if extra['policy'] == 'random':
RUNNER_HOSTS.set_random_host(response)
else:
RUNNER_HOSTS.set_closest_host(response)
stream.close()
def handle_end_of_body(self, stream):
# Note: this function MUST be callable multiple times
context = stream.opaque
extra = context.extra
if extra['requests'] <= 0:
raise RuntimeError('skype_negotiate: unexpected response')
extra['requests'] -= 1
tmp = context.headers.get(CONTENT_TYPE)
if context.code != CODE200 or tmp != APPLICATION_JSON:
logging.error('skype_negotiate: bad response')
stream.close()
return
response = json.loads(six.u(extra['body'].getvalue()))
http_utils.prettyprint_json(response, '<')
if STATE.current == 'negotiate':
self._process_negotiate_response(stream, response)
elif STATE.current == 'collect':
self._process_collect_response(stream, response)
else:
raise RuntimeError('skype_negotiate: internal error')
def _rewrite_response(request, response):
''' Rewrite response and translate JSON to XML '''
# Do not touch error responses
if response.code != '200':
return
# Convert JSON response to XML
elif request.uri == '/negotiate/speedtest':
response_body = json.loads(response.body)
xmlresp = SpeedtestNegotiate_Response()
xmlresp.authorization = response_body['authorization']
xmlresp.unchoked = response_body['unchoked']
xmlresp.queuePos = response_body['queue_pos']
xmlresp.publicAddress = response_body['real_address']
response.body = marshal.marshal_object(xmlresp, 'application/xml')
del response['content-type']
del response['content-length']
response['content-type'] = 'application/xml'
response['content-length'] = str(len(response.body))
# Suppress JSON response
elif request.uri == '/collect/speedtest':
del response['content-type']
del response['content-length']
response.body = ''
#
# We MUST NOT be too strict here because old clients
# use the same stream for both negotiation and testing
# and the stream already has the rewriter installed
# due to that.
# Probably we can remove the rewrite hook just after
# usage and be more strict here, but the current code
# seems to me more robust.
#
else:
pass
def process_request(self, stream, request):
m = {
"code": "200",
"ident": sha1stream(stream),
"keepalive": True,
"mimetype": "application/json",
"reason": "Ok",
"request_body": request.body.read(),
"request": request,
"response_body": "",
"parent": self,
"stream": stream,
}
# We expect a JSONized dictionary or nothing
if m["request_body"]:
if request["content-type"] != "application/json":
raise RuntimeError("Invalid MIME type")
m["request_body"] = dict(json.loads(m["request_body"]))
else:
m["request_body"] = {}
if request.uri.startswith("/negotiate/"):
m["module"] = request.uri.replace("/negotiate/", "")
self.negotiator.negotiate(m)
# NO because negotiate can use comet
#self.send_response(m)
elif request.uri.startswith("/collect/"):
m["module"] = request.uri.replace("/collect/", "")
self.negotiator.collect(m)
self.send_response(m)
else:
m["code"] = "404"
m["keepalive"] = False
m["mimetype"] = "text/plain"
m["reason"] = "Not Found"
m["response_body"] = "Not Found"
self.send_response(m)
def test_negotiate_successful(self):
''' Make sure the response is OK when negotiate succeeds '''
server = NegotiateServer(None)
server.register_module('abc', NegotiateServerModule())
# Want to check authorized and nonauthorized streams
for position in range(CONFIG['negotiate.parallelism'] + 3):
stream = MinimalHttpStream()
request = Message(uri='/negotiate/abc')
request.body = StringIO.StringIO('{}')
server.process_request(stream, request)
response = stream.response
self.assertEqual(response.code, '200')
self.assertEqual(response.reason, 'Ok')
self.assertNotEqual(response['connection'], 'close')
self.assertEqual(response['content-type'], 'application/json')
# Note: authorization is empty when you're choked
body = json.loads(response.body)
if position < CONFIG['negotiate.parallelism']:
self.assertEqual(
body, {
u'unchoked': 1,
u'queue_pos': position,
u'real_address': u'abc',
u'authorization': unicode(hash(stream))
})
else:
self.assertEqual(
body, {
u'unchoked': 0,
u'queue_pos': position,
u'real_address': u'abc',
u'authorization': u'',
})
def test_negotiate_successful(self):
''' Make sure the response is OK when negotiate succeeds '''
server = NegotiateServer(None)
server.register_module('abc', NegotiateServerModule())
# Want to check authorized and nonauthorized streams
for position in range(CONFIG['negotiate.parallelism'] + 3):
stream = MinimalHttpStream()
request = Message(uri='/negotiate/abc')
request.body = StringIO.StringIO('{}')
server.process_request(stream, request)
response = stream.response
self.assertEqual(response.code, '200')
self.assertEqual(response.reason, 'Ok')
self.assertNotEqual(response['connection'], 'close')
self.assertEqual(response['content-type'], 'application/json')
# Note: authorization is empty when you're choked
body = json.loads(response.body)
if position < CONFIG['negotiate.parallelism']:
self.assertEqual(body, {
u'unchoked': 1,
u'queue_pos': position,
u'real_address': u'abc',
u'authorization': unicode(hash(stream))
})
else:
self.assertEqual(body, {
u'unchoked': 0,
u'queue_pos': position,
u'real_address': u'abc',
u'authorization': u'',
})
def main(args):
''' main function '''
try:
options, arguments = getopt.getopt(args[1:], 'b:d:Fm:t:u:v')
except getopt.error:
sys.exit(USAGE)
# Good-enough default message
default_msg = {
"timestamp": 0,
"uuid": "",
"internal_address": "",
"real_address": "",
"remote_address": "",
"privacy_informed": 0,
"privacy_can_collect": 0,
"privacy_can_publish": 0,
"latency": 0.0,
"connect_time": 0.0,
"download_speed": 0.0,
"upload_speed": 0.0,
"neubot_version": "",
"platform": "",
}
bcknd = None
datadir = None
filesys_mode = False
msg = default_msg
timestamp = None
uname = None
for name, value in options:
if name == '-b':
bcknd = value
elif name == '-d':
datadir = value
elif name == "-F":
filesys_mode = True
elif name == '-m':
msg = value
elif name == '-t':
timestamp = float(value)
elif name == "-u":
uname = value
elif name == '-v':
CONFIG['verbose'] = 1
if bcknd:
BACKEND.use_backend(bcknd)
if msg != default_msg:
msg = json.loads(msg)
if timestamp:
time.time = lambda: timestamp # XXX
BACKEND.datadir_init(uname=uname, datadir=datadir)
if filesys_mode and arguments:
BACKEND.datadir_touch(arguments)
sys.exit(0)
if arguments:
sys.exit(USAGE)
BACKEND.bittorrent_store(msg)
BACKEND.speedtest_store(msg)
BACKEND.store_raw(msg)
BACKEND.store_generic("generictest", msg)
def json_to_dictionary(s):
return dict(json.loads(s))
