def test_done(self, *baton):
''' Invoked when the test is done '''
#
# Stop streaming test events to interested parties
# via the log streaming API.
#
LOG.stop_streaming()
# Paranoid
if baton[0] != 'testdone':
raise RuntimeError('Invoked for the wrong event')
# Notify the caller that the test is done
callback = self.queue.popleft()[2]
callback()
#
# Allow for more tests
# If callback() adds one more test, that would
# be run by the run_queue() invocation below.
#
self.running = False
# Eventually run next queued test
self.run_queue()
def connection_made(self, sock, rtt=0):
if rtt:
LOG.debug("ClientHTTP: latency: %s" % utils.time_formatter(rtt))
self.rtt = rtt
stream = ClientStream(self.poller)
stream.attach(self, sock, self.conf, self.measurer)
self.connection_ready(stream)
def read_send_queue(self):
octets = ""
while self.send_queue:
octets = self.send_queue[0]
if isinstance(octets, basestring):
# remove the piece in any case
self.send_queue.popleft()
if octets:
break
else:
octets = octets.read(MAXBUF)
if octets:
break
# remove the file-like when it is empty
self.send_queue.popleft()
if octets:
if type(octets) == types.UnicodeType:
LOG.oops("Received unicode input")
octets = octets.encode("utf-8")
if self.encrypt:
octets = self.encrypt(octets)
return octets
def lookup_country(self, address):
''' Lookup for country entry '''
country = self.countries.country_code_by_addr(address)
if not country:
LOG.error("Geolocator: %s: not found" % address)
return ""
return utils.stringify(country)
def main(args):
''' Main() function '''
try:
options, arguments = getopt.getopt(args[1:], '')
except getopt.error:
sys.exit('usage: neubot background_win32')
if options or arguments:
sys.exit('usage: neubot background_win32')
# Read settings from database
CONFIG.merge_database(DATABASE.connection())
#
# Save logs into the database, to easily access
# and show them via the web user interface.
#
LOG.use_database()
# Complain if privacy settings are not OK
privacy.complain_if_needed()
background_api.start('127.0.0.1 ::1', '9774')
BACKGROUND_RENDEZVOUS.start()
__start_updater()
POLLER.loop()
def main(args):
''' main() function '''
try:
options, arguments = getopt.getopt(args[1:], 'vy')
except getopt.error:
sys.exit('neubot updater_runner [-vy] [version]')
if len(arguments) > 1:
sys.exit('neubot updater_runner [-vy] [version]')
privacy = False
for tpl in options:
if tpl[0] == '-v':
LOG.verbose()
elif tpl[0] == '-y':
privacy = True
# Honor -y and force privacy permissions
if privacy:
CONFIG.conf.update({'privacy.informed': 1, 'privacy.can_collect': 1,
'privacy.can_publish': 1})
updater = UpdaterRunner('win32', os.path.dirname(ROOTDIR))
if arguments:
updater.retrieve_files(arguments[0])
else:
# Enable automatic updates if we arrive here
CONFIG.conf['win32_updater'] = 1
updater.retrieve_versioninfo()
POLLER.loop()
def run_queue(self):
''' If possible run the first test in queue '''
# Adapted from neubot/rendezvous/client.py
if not self.queue:
return
if self.running:
return
#
# Subscribe BEFORE starting the test, otherwise we
# may miss the 'testdone' event if the connection
# to the negotiator service fails, and we will stay
# stuck forever.
#
NOTIFIER.subscribe('testdone', self.test_done)
# Prevent concurrent tests
self.running = True
# Safely run first element in queue
try:
self._do_run_queue()
except (SystemExit, KeyboardInterrupt):
raise
except:
exc = asyncore.compact_traceback()
error = str(exc)
LOG.error('runner_core: catched exception: %s' % error)
NOTIFIER.publish('testdone')
def got_response_collecting(self, stream, request, response):
LOG.complete()
if self.success:
#
# Always measure at the receiver because there is more
# information at the receiver and also to make my friend
# Enrico happier :-P.
# The following is not a bug: it's just that the server
# returns a result using the point of view of the client,
# i.e. upload_speed is _our_ upload speed.
#
m = json.loads(response.body.read())
self.my_side["upload_speed"] = m["upload_speed"]
upload = utils.speed_formatter(m["upload_speed"])
STATE.update("test_upload", upload)
if privacy.collect_allowed(self.my_side):
table_bittorrent.insert(DATABASE.connection(), self.my_side)
# Update the upstream channel estimate
target_bytes = int(m["target_bytes"])
if target_bytes > 0:
estimate.UPLOAD = target_bytes
stream.close()
def main(args):
CONFIG.register_descriptions({
"speedtest.negotiate.address": "Address to listen to",
"speedtest.negotiate.auth_only": "Enable doing tests for authorized clients only",
"speedtest.negotiate.daemonize": "Enable going in background",
"speedtest.negotiate.port": "Port to listen to",
})
common.main("speedtest.negotiate", "Speedtest negotiation server", args)
conf = CONFIG.copy()
server = ServerSpeedtest(POLLER)
server.configure(conf)
server.listen((conf["speedtest.negotiate.address"],
conf["speedtest.negotiate.port"]))
if conf["speedtest.negotiate.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
''' Run a subcommand's main() '''
# Args[0] must be the subcommand name
subcommand = args[0]
# Users are not supposed to prefix commands with 'neubot.'
subcommand = 'neubot.' + subcommand
# Dinamically load the selected subcommand's main() at runtime
__import__(subcommand)
mainfunc = sys.modules[subcommand].main
# Fix args[0]
args[0] = 'neubot ' + subcommand
# Run main()
try:
mainfunc(args)
except KeyboardInterrupt:
sys.exit(1)
except SystemExit:
raise
except:
LOG.exception()
sys.exit(1)
def merge_api(self, dictlike, database=None):
# enforce all-or-nothing
LOG.debug("config: reading properties from /api/config")
map(lambda t: self.merge_kv(t, dry=True), dictlike.iteritems())
map(self.merge_kv, dictlike.iteritems())
if database:
table_config.update(database, dictlike.iteritems())
def main():
''' Merge Neubot databases '''
syslog.openlog('merge.py', syslog.LOG_PERROR, syslog.LOG_USER)
output = 'database.sqlite3'
try:
options, arguments = getopt.getopt(sys.argv[1:], 'o:v')
except getopt.error:
sys.exit('Usage: merge.py [-v] [-o output] file...')
if not arguments:
sys.exit('Usage: merge.py [-v] [-o output] file...')
for name, value in options:
if name == '-o':
output = value
elif value == '-v':
LOG.verbose()
beginning = {}
destination = __sqlite3_connect(output)
for argument in arguments:
source = __sqlite3_connect(argument)
for table in ('speedtest', 'bittorrent'):
# Just in case there are overlapping measurements
beginning[table] = __lookup_last(destination, table)
__copy_table(source, destination, table, beginning[table])
destination.commit()
def send_response(self, request, response):
''' Send a response to the client '''
if self.response_rewriter:
self.response_rewriter(request, response)
if request['connection'] == 'close' or request.protocol == 'HTTP/1.0':
del response['connection']
response['connection'] = 'close'
self.send_message(response)
if response['connection'] == 'close':
self.close()
address = self.peername[0]
now = time.gmtime()
timestring = "%02d/%s/%04d:%02d:%02d:%02d -0000" % (now.tm_mday,
MONTH[now.tm_mon], now.tm_year, now.tm_hour, now.tm_min, now.tm_sec)
requestline = request.requestline
statuscode = response.code
nbytes = "-"
if response["content-length"]:
nbytes = response["content-length"]
if nbytes == "0":
nbytes = "-"
LOG.log("ACCESS",
"%s - - [%s] \"%s\" %s %s",
(address, timestring, requestline, statuscode, nbytes),
None)
def _send_handshake(self):
''' Convenience function to send handshake '''
LOG.debug("> HANDSHAKE infohash=%s id=%s" %
(self.parent.infohash.encode("hex"),
self.parent.my_id.encode("hex")))
self.start_send("".join((chr(len(PROTOCOL_NAME)), PROTOCOL_NAME,
FLAGS, self.parent.infohash, self.parent.my_id)))
def main(args):
''' Run the API server '''
try:
options, arguments = getopt.getopt(args[1:], 'O:v')
except getopt.error:
sys.exit('usage: neubot background_api [-v] [-O setting]')
if arguments:
sys.exit('usage: neubot background_api [-v] [-O setting]')
settings = []
for name, value in options:
if name == '-O':
settings.append(value)
elif name == '-v':
LOG.verbose()
settings = utils_rc.parse_safe(iterable=settings)
if not 'address' in settings:
settings['address'] = '127.0.0.1 ::1'
if not 'port' in settings:
settings['port'] = '9774'
start(settings['address'], settings['port'])
POLLER.loop()
def _on_internal_error(self, stream, request):
LOG.exception()
response = Message()
response.compose(code="500", reason="Internal Server Error",
body="500 Internal Server Error", keepalive=0)
stream.send_response(request, response)
stream.close()
def main(args):
CONFIG.register_descriptions({
"rendezvous.server.address": "Set rendezvous server address",
"rendezvous.server.daemonize": "Enable daemon behavior",
"rendezvous.server.ports": "List of rendezvous server ports",
"rendezvous.server.update_uri": "Where to download updates from",
"rendezvous.server.update_version": "Update Neubot version number",
"rendezvous.geoip_wrapper.country_database": "Path of the GeoIP country database",
"rendezvous.server.default": "Default test server to use",
})
common.main("rendezvous.server", "Rendezvous server", args)
conf = CONFIG.copy()
HTTP_SERVER.configure(conf)
for port in conf["rendezvous.server.ports"].split(","):
HTTP_SERVER.listen((conf["rendezvous.server.address"], int(port)))
# Really start this module
run(POLLER, conf)
if conf["rendezvous.server.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def attach(self, parent, sock, conf):
self.parent = parent
self.conf = conf
self.filenum = sock.fileno()
self.myname = sock.getsockname()
self.peername = sock.getpeername()
self.logname = str((self.myname, self.peername))
LOG.debug("* Connection made %s" % str(self.logname))
if conf["net.stream.secure"]:
if not ssl:
raise RuntimeError("SSL support not available")
server_side = conf["net.stream.server_side"]
certfile = conf["net.stream.certfile"]
# wrap_socket distinguishes between None and ''
if not certfile:
certfile = None
ssl_sock = ssl.wrap_socket(sock, do_handshake_on_connect=False,
certfile=certfile, server_side=server_side)
self.sock = SSLWrapper(ssl_sock)
self.recv_ssl_needs_kickoff = not server_side
else:
self.sock = SocketWrapper(sock)
self.connection_made()
def main(args):
''' Main function '''
try:
options, arguments = getopt.getopt(args[1:], 'D:t:v')
except getopt.error:
sys.exit('usage: notifier_browser [-v] [-D setting] [-t time] page...')
if not arguments:
sys.exit('usage: notifier_browser [-v] [-D setting] [-t time] page...')
sleeptime = 0
for name, value in options:
if name == '-D':
CONFIG.register_property(value)
elif name == '-t':
sleeptime = int(value)
elif name == '-v':
LOG.verbose()
CONFIG.merge_properties()
for argument in arguments:
if argument == 'privacy':
NOTIFIER_BROWSER.notify_bad_privacy()
elif argument == 'update':
NOTIFIER_BROWSER.notify_update_avail()
else:
sys.exit('Invalid page. Valid pages are: privacy, update')
if sleeptime:
logging.debug('notifier_browser: sleep for %d seconds', sleeptime)
time.sleep(sleeptime)
def prettyprintbody(self, prefix):
''' Pretty print body '''
if self["content-type"] not in ("application/json", "text/xml",
"application/xml"):
return
# Grab the whole body
if not isinstance(self.body, basestring):
body = self.body.read()
else:
body = self.body
# Decode the body
if self["content-type"] == "application/json":
string = compat.json.dumps(compat.json.loads(body),
indent=4, sort_keys=True)
elif self["content-type"] in ("text/xml", "application/xml"):
string = body
# Prettyprint
for line in string.split("\n"):
LOG.debug("%s %s" % (prefix, line.rstrip()))
# Seek to the beginning if needed
if not isinstance(self.body, basestring):
utils.safe_seek(self.body, 0)
def connection_ready(self, stream):
''' Invoked when the connection is ready '''
method = self.conf["http.client.method"]
stdout = self.conf["http.client.stdout"]
uri = self.conf["http.client.uri"]
request = Message()
if method == "PUT":
fpath = uri.split("/")[-1]
if not os.path.exists(fpath):
LOG.error("* Local file does not exist: %s" % fpath)
sys.exit(1)
request.compose(method=method, uri=uri, keepalive=False,
mimetype="text/plain", body=open(fpath, "rb"))
else:
request.compose(method=method, uri=uri, keepalive=False)
response = Message()
if method == "GET" and not stdout:
fpath = uri.split("/")[-1]
if os.path.exists(fpath):
LOG.error("* Local file already exists: %s" % fpath)
sys.exit(1)
response.body = open(fpath, "wb")
else:
response.body = sys.stdout
stream.send_request(request, response)
def check_response(self, response):
if response.code != "200":
raise ValueError("Bad HTTP response code")
if response["content-type"] != "application/json":
raise ValueError("Unexpected contenty type")
octets = response.body.read()
dictionary = json.loads(octets)
LOG.debug("APIStateTracker: received JSON: " +
json.dumps(dictionary, ensure_ascii=True))
if not "events" in dictionary:
return
if not "current" in dictionary:
raise ValueError("Incomplete dictionary")
t = dictionary["t"]
if not type(t) == types.IntType and not type(t) == types.LongType:
raise ValueError("Invalid type for current event time")
if t < 0:
raise ValueError("Invalid value for current event time")
self.timestamp = t
self.process_dictionary(dictionary)
def _update_queue(self, lost_stream, ignored):
''' Invoked when a connection is lost '''
queue, found = collections.deque(), False
position = 0
for stream in self.queue:
if not found:
if lost_stream != stream:
position += 1
queue.append(stream)
else:
found = True
self.known.remove(stream)
elif not stream.opaque:
position += 1
queue.append(stream)
else:
request, stream.opaque = stream.opaque, None
try:
self._do_negotiate((stream, request, position))
position += 1
queue.append(stream)
except (KeyboardInterrupt, SystemExit):
raise
except:
LOG.exception()
stream.unregister_atclose(self._update_queue)
self.known.remove(stream)
stream.close()
self.queue = queue
def open_or_die(self):
''' Open the database or die '''
if not GEOIP:
LOG.error("Missing dependency: GeoIP")
LOG.info("Please install GeoIP python wrappers, e.g.")
LOG.info(" sudo apt-get install python-geoip")
sys.exit(1)
path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
COUNTRY_DATABASE)
#
# Detect the common error case, i.e. that the user has
# not downloaded the database. If something fancy is
# going on, let the GeoIP library stacktrace for us.
#
if not os.path.exists(path):
LOG.error("Missing GeoLiteCountry database: %s" % path)
LOG.info("Please download it from "
"<http://www.maxmind.com/app/geolitecountry>.")
sys.exit(1)
self.countries = GEOIP.open(path, GEOIP.GEOIP_STANDARD)
def main(args):
CONFIG.register_descriptions({
"speedtest.client.uri": "Base URI to connect to",
"speedtest.client.nconn": "Number of concurrent connections to use",
"speedtest.client.latency_tries": "Number of latency measurements",
})
common.main("speedtest.client", "Speedtest client", args)
conf = CONFIG.copy()
#
# If possible use the runner, which will execute the
# test in the context of the neubot daemon. Then exit
# to bypass the POLLER.loop() invokation that is below
# here.
# If the runner fails, fallback to the usual code path,
# which executes the test in the context of the local
# process.
# Set 'runned.enabled' to 0 to bypass the runner and
# run the test locally.
#
if (utils.intify(conf['runner.enabled']) and
runner_clnt.runner_client(conf["agent.api.address"],
conf["agent.api.port"],
LOG.noisy, "speedtest")):
sys.exit(0)
LOG.info('Will run the test in the local context...')
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
daemonize = True
blink = False
nohide = True
try:
options, arguments = getopt.getopt(args[1:], "BdnqVv", ["help"])
except getopt.GetoptError:
sys.stderr.write(USAGE % args[0])
sys.exit(1)
for name, value in options:
if name == "-B":
blink = True
elif name == "-d":
daemonize = False
elif name == "--help":
sys.stdout.write(HELP % args[0])
sys.exit(0)
elif name == "-n":
nohide = True
elif name == "-q":
nohide = False
elif name == "-V":
sys.stderr.write(VERSION + "\n")
sys.exit(0)
elif name == "-v":
LOG.verbose()
if len(arguments) >= 3:
sys.stderr.write(USAGE % args[0])
sys.exit(1)
elif len(arguments) == 2:
address = arguments[0]
port = arguments[1]
elif len(arguments) == 1:
address = ADDRESS
port = arguments[0]
else:
address = ADDRESS
port = PORT
if daemonize:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
gtk.gdk.threads_init()
icon = StatusIcon(address, port, blink, nohide)
tracker = StateTrackerThread(icon, address, port)
tracker.daemon = True
tracker.start()
gtk.gdk.threads_enter()
gtk.main()
gtk.gdk.threads_leave()
tracker.interrupt()
def connection_lost(self, stream):
if NOTIFIER.is_subscribed("testdone"):
LOG.debug("RendezVous: don't _schedule(): test in progress")
return
if self._task:
LOG.debug("RendezVous: don't _schedule(): we already have a task")
return
self._schedule()
def connect_uri(self, uri=None, count=None):
if not uri:
uri = self.conf.get("speedtest.client.uri",
"http://master.neubot.org/")
if not count:
count = self.conf.get("speedtest.client.nconn", 1)
LOG.info("* speedtest with %s" % uri)
ClientHTTP.connect_uri(self, uri, count)
def connection_lost(self, stream):
if runner_core.test_is_running():
LOG.debug("RendezVous: don't _schedule(): test in progress")
return
if self._task:
LOG.debug("RendezVous: don't _schedule(): we already have a task")
return
self._schedule()
def connection_made(self, sock, rtt=0):
''' Invoked when the connection is created '''
if rtt:
LOG.debug("ClientHTTP: latency: %s" % utils.time_formatter(rtt))
self.rtt = rtt
stream = ClientStream(self.poller)
stream.attach(self, sock, self.conf)
self.connection_ready(stream)