Example #1
0
    def test_done(self, *baton):
        ''' Invoked when the test is done '''

        #
        # Stop streaming test events to interested parties
        # via the log streaming API.
        #
        LOG.stop_streaming()

        # Paranoid
        if baton[0] != 'testdone':
            raise RuntimeError('Invoked for the wrong event')

        # Notify the caller that the test is done
        callback = self.queue.popleft()[2]
        callback()

        #
        # Allow for more tests
        # If callback() adds one more test, that would
        # be run by the run_queue() invocation below.
        #
        self.running = False

        # Eventually run next queued test
        self.run_queue()
Example #2
0
 def connection_made(self, sock, rtt=0):
     if rtt:
         LOG.debug("ClientHTTP: latency: %s" % utils.time_formatter(rtt))
         self.rtt = rtt
     stream = ClientStream(self.poller)
     stream.attach(self, sock, self.conf, self.measurer)
     self.connection_ready(stream)
Example #3
0
    def read_send_queue(self):
        octets = ""

        while self.send_queue:
            octets = self.send_queue[0]
            if isinstance(octets, basestring):
                # remove the piece in any case
                self.send_queue.popleft()
                if octets:
                    break
            else:
                octets = octets.read(MAXBUF)
                if octets:
                    break
                # remove the file-like when it is empty
                self.send_queue.popleft()

        if octets:
            if type(octets) == types.UnicodeType:
                LOG.oops("Received unicode input")
                octets = octets.encode("utf-8")
            if self.encrypt:
                octets = self.encrypt(octets)

        return octets
Example #4
0
 def lookup_country(self, address):
     ''' Lookup for country entry '''
     country = self.countries.country_code_by_addr(address)
     if not country:
         LOG.error("Geolocator: %s: not found" % address)
         return ""
     return utils.stringify(country)
def main(args):
    ''' Main() function '''

    try:
        options, arguments = getopt.getopt(args[1:], '')
    except getopt.error:
        sys.exit('usage: neubot background_win32')
    if options or arguments:
        sys.exit('usage: neubot background_win32')

    # Read settings from database
    CONFIG.merge_database(DATABASE.connection())

    #
    # Save logs into the database, to easily access
    # and show them via the web user interface.
    #
    LOG.use_database()

    # Complain if privacy settings are not OK
    privacy.complain_if_needed()

    background_api.start('127.0.0.1 ::1', '9774')
    BACKGROUND_RENDEZVOUS.start()

    __start_updater()

    POLLER.loop()
Example #6
0
def main(args):
    ''' main() function '''

    try:
        options, arguments = getopt.getopt(args[1:], 'vy')
    except getopt.error:
        sys.exit('neubot updater_runner [-vy] [version]')
    if len(arguments) > 1:
        sys.exit('neubot updater_runner [-vy] [version]')

    privacy = False
    for tpl in options:
        if tpl[0] == '-v':
            LOG.verbose()
        elif tpl[0] == '-y':
            privacy = True

    # Honor -y and force privacy permissions
    if privacy:
        CONFIG.conf.update({'privacy.informed': 1, 'privacy.can_collect': 1,
                            'privacy.can_publish': 1})

    updater = UpdaterRunner('win32', os.path.dirname(ROOTDIR))

    if arguments:
        updater.retrieve_files(arguments[0])
    else:
        # Enable automatic updates if we arrive here
        CONFIG.conf['win32_updater'] = 1
        updater.retrieve_versioninfo()

    POLLER.loop()
Example #7
0
    def run_queue(self):
        ''' If possible run the first test in queue '''

        # Adapted from neubot/rendezvous/client.py

        if not self.queue:
            return
        if self.running:
            return

        #
        # Subscribe BEFORE starting the test, otherwise we
        # may miss the 'testdone' event if the connection
        # to the negotiator service fails, and we will stay
        # stuck forever.
        #
        NOTIFIER.subscribe('testdone', self.test_done)

        # Prevent concurrent tests
        self.running = True

        # Safely run first element in queue
        try:
            self._do_run_queue()
        except (SystemExit, KeyboardInterrupt):
            raise
        except:
            exc = asyncore.compact_traceback()
            error = str(exc)
            LOG.error('runner_core: catched exception: %s' % error)
            NOTIFIER.publish('testdone')
Example #8
0
    def got_response_collecting(self, stream, request, response):
        LOG.complete()

        if self.success:
            #
            # Always measure at the receiver because there is more
            # information at the receiver and also to make my friend
            # Enrico happier :-P.
            # The following is not a bug: it's just that the server
            # returns a result using the point of view of the client,
            # i.e. upload_speed is _our_ upload speed.
            #
            m = json.loads(response.body.read())
            self.my_side["upload_speed"] = m["upload_speed"]

            upload = utils.speed_formatter(m["upload_speed"])
            STATE.update("test_upload", upload)

            if privacy.collect_allowed(self.my_side):
                table_bittorrent.insert(DATABASE.connection(), self.my_side)

            # Update the upstream channel estimate
            target_bytes = int(m["target_bytes"])
            if target_bytes > 0:
                estimate.UPLOAD = target_bytes

        stream.close()
Example #9
0
def main(args):

    CONFIG.register_descriptions({
        "speedtest.negotiate.address": "Address to listen to",
        "speedtest.negotiate.auth_only": "Enable doing tests for authorized clients only",
        "speedtest.negotiate.daemonize": "Enable going in background",
        "speedtest.negotiate.port": "Port to listen to",
    })

    common.main("speedtest.negotiate", "Speedtest negotiation server", args)

    conf = CONFIG.copy()

    server = ServerSpeedtest(POLLER)
    server.configure(conf)
    server.listen((conf["speedtest.negotiate.address"],
                  conf["speedtest.negotiate.port"]))

    if conf["speedtest.negotiate.daemonize"]:
        system.change_dir()
        system.go_background()
        LOG.redirect()

    system.drop_privileges(LOG.error)
    POLLER.loop()
Example #10
0
def main(args):
    ''' Run a subcommand's main() '''

    # Args[0] must be the subcommand name
    subcommand = args[0]

    # Users are not supposed to prefix commands with 'neubot.'
    subcommand = 'neubot.' + subcommand

    # Dinamically load the selected subcommand's main() at runtime
    __import__(subcommand)
    mainfunc = sys.modules[subcommand].main

    # Fix args[0]
    args[0] = 'neubot ' + subcommand

    # Run main()
    try:
        mainfunc(args)
    except KeyboardInterrupt:
        sys.exit(1)
    except SystemExit:
        raise
    except:
        LOG.exception()
        sys.exit(1)
Example #11
0
 def merge_api(self, dictlike, database=None):
     # enforce all-or-nothing
     LOG.debug("config: reading properties from /api/config")
     map(lambda t: self.merge_kv(t, dry=True), dictlike.iteritems())
     map(self.merge_kv, dictlike.iteritems())
     if database:
         table_config.update(database, dictlike.iteritems())
Example #12
0
def main():

    ''' Merge Neubot databases '''

    syslog.openlog('merge.py', syslog.LOG_PERROR, syslog.LOG_USER)
    output = 'database.sqlite3'

    try:
        options, arguments = getopt.getopt(sys.argv[1:], 'o:v')
    except getopt.error:
        sys.exit('Usage: merge.py [-v] [-o output] file...')
    if not arguments:
        sys.exit('Usage: merge.py [-v] [-o output] file...')

    for name, value in options:
        if name == '-o':
            output = value
        elif value == '-v':
            LOG.verbose()

    beginning = {}
    destination = __sqlite3_connect(output)
    for argument in arguments:
        source = __sqlite3_connect(argument)
        for table in ('speedtest', 'bittorrent'):
            # Just in case there are overlapping measurements
            beginning[table] = __lookup_last(destination, table)
            __copy_table(source, destination, table, beginning[table])

    destination.commit()
Example #13
0
    def send_response(self, request, response):
        ''' Send a response to the client '''

        if self.response_rewriter:
            self.response_rewriter(request, response)

        if request['connection'] == 'close' or request.protocol == 'HTTP/1.0':
            del response['connection']
            response['connection'] = 'close'

        self.send_message(response)

        if response['connection'] == 'close':
            self.close()

        address = self.peername[0]
        now = time.gmtime()
        timestring = "%02d/%s/%04d:%02d:%02d:%02d -0000" % (now.tm_mday,
          MONTH[now.tm_mon], now.tm_year, now.tm_hour, now.tm_min, now.tm_sec)
        requestline = request.requestline
        statuscode = response.code

        nbytes = "-"
        if response["content-length"]:
            nbytes = response["content-length"]
            if nbytes == "0":
                nbytes = "-"

        LOG.log("ACCESS",
                "%s - - [%s] \"%s\" %s %s",
                (address, timestring, requestline, statuscode, nbytes),
                None)
Example #14
0
 def _send_handshake(self):
     ''' Convenience function to send handshake '''
     LOG.debug("> HANDSHAKE infohash=%s id=%s" %
               (self.parent.infohash.encode("hex"),
                self.parent.my_id.encode("hex")))
     self.start_send("".join((chr(len(PROTOCOL_NAME)), PROTOCOL_NAME,
       FLAGS, self.parent.infohash, self.parent.my_id)))
Example #15
0
def main(args):
    ''' Run the API server '''

    try:
        options, arguments = getopt.getopt(args[1:], 'O:v')
    except getopt.error:
        sys.exit('usage: neubot background_api [-v] [-O setting]')
    if arguments:
        sys.exit('usage: neubot background_api [-v] [-O setting]')

    settings = []
    for name, value in options:
        if name == '-O':
            settings.append(value)
        elif name == '-v':
            LOG.verbose()

    settings = utils_rc.parse_safe(iterable=settings)
    if not 'address' in settings:
        settings['address'] = '127.0.0.1 ::1'
    if not 'port' in settings:
        settings['port'] = '9774'

    start(settings['address'], settings['port'])
    POLLER.loop()
Example #16
0
 def _on_internal_error(self, stream, request):
     LOG.exception()
     response = Message()
     response.compose(code="500", reason="Internal Server Error",
                      body="500 Internal Server Error", keepalive=0)
     stream.send_response(request, response)
     stream.close()
Example #17
0
def main(args):

    CONFIG.register_descriptions({
        "rendezvous.server.address": "Set rendezvous server address",
        "rendezvous.server.daemonize": "Enable daemon behavior",
        "rendezvous.server.ports": "List of rendezvous server ports",
        "rendezvous.server.update_uri": "Where to download updates from",
        "rendezvous.server.update_version": "Update Neubot version number",
        "rendezvous.geoip_wrapper.country_database": "Path of the GeoIP country database",
        "rendezvous.server.default": "Default test server to use",
    })

    common.main("rendezvous.server", "Rendezvous server", args)
    conf = CONFIG.copy()

    HTTP_SERVER.configure(conf)
    for port in conf["rendezvous.server.ports"].split(","):
        HTTP_SERVER.listen((conf["rendezvous.server.address"], int(port)))

    # Really start this module
    run(POLLER, conf)

    if conf["rendezvous.server.daemonize"]:
        system.change_dir()
        system.go_background()
        LOG.redirect()

    system.drop_privileges(LOG.error)
    POLLER.loop()
Example #18
0
    def attach(self, parent, sock, conf):

        self.parent = parent
        self.conf = conf

        self.filenum = sock.fileno()
        self.myname = sock.getsockname()
        self.peername = sock.getpeername()
        self.logname = str((self.myname, self.peername))

        LOG.debug("* Connection made %s" % str(self.logname))

        if conf["net.stream.secure"]:
            if not ssl:
                raise RuntimeError("SSL support not available")

            server_side = conf["net.stream.server_side"]
            certfile = conf["net.stream.certfile"]

            # wrap_socket distinguishes between None and ''
            if not certfile:
                certfile = None

            ssl_sock = ssl.wrap_socket(sock, do_handshake_on_connect=False,
              certfile=certfile, server_side=server_side)
            self.sock = SSLWrapper(ssl_sock)

            self.recv_ssl_needs_kickoff = not server_side

        else:
            self.sock = SocketWrapper(sock)

        self.connection_made()
Example #19
0
def main(args):
    ''' Main function '''

    try:
        options, arguments = getopt.getopt(args[1:], 'D:t:v')
    except getopt.error:
        sys.exit('usage: notifier_browser [-v] [-D setting] [-t time] page...')
    if not arguments:
        sys.exit('usage: notifier_browser [-v] [-D setting] [-t time] page...')

    sleeptime = 0
    for name, value in options:
        if name == '-D':
            CONFIG.register_property(value)
        elif name == '-t':
            sleeptime = int(value)
        elif name == '-v':
            LOG.verbose()

    CONFIG.merge_properties()

    for argument in arguments:
        if argument == 'privacy':
            NOTIFIER_BROWSER.notify_bad_privacy()
        elif argument == 'update':
            NOTIFIER_BROWSER.notify_update_avail()
        else:
            sys.exit('Invalid page.  Valid pages are: privacy, update')

        if sleeptime:
            logging.debug('notifier_browser: sleep for %d seconds', sleeptime)
            time.sleep(sleeptime)
Example #20
0
    def prettyprintbody(self, prefix):
        ''' Pretty print body '''
        if self["content-type"] not in ("application/json", "text/xml",
                                        "application/xml"):
            return

        # Grab the whole body
        if not isinstance(self.body, basestring):
            body = self.body.read()
        else:
            body = self.body

        # Decode the body
        if self["content-type"] == "application/json":
            string = compat.json.dumps(compat.json.loads(body),
              indent=4, sort_keys=True)
        elif self["content-type"] in ("text/xml", "application/xml"):
            string = body

        # Prettyprint
        for line in string.split("\n"):
            LOG.debug("%s %s" % (prefix, line.rstrip()))

        # Seek to the beginning if needed
        if not isinstance(self.body, basestring):
            utils.safe_seek(self.body, 0)
Example #21
0
    def connection_ready(self, stream):
        ''' Invoked when the connection is ready '''
        method = self.conf["http.client.method"]
        stdout = self.conf["http.client.stdout"]
        uri = self.conf["http.client.uri"]

        request = Message()
        if method == "PUT":
            fpath = uri.split("/")[-1]
            if not os.path.exists(fpath):
                LOG.error("* Local file does not exist: %s" % fpath)
                sys.exit(1)
            request.compose(method=method, uri=uri, keepalive=False,
              mimetype="text/plain", body=open(fpath, "rb"))
        else:
            request.compose(method=method, uri=uri, keepalive=False)

        response = Message()
        if method == "GET" and not stdout:
            fpath = uri.split("/")[-1]
            if os.path.exists(fpath):
                LOG.error("* Local file already exists: %s" % fpath)
                sys.exit(1)
            response.body = open(fpath, "wb")
        else:
            response.body = sys.stdout

        stream.send_request(request, response)
Example #22
0
    def check_response(self, response):

        if response.code != "200":
            raise ValueError("Bad HTTP response code")
        if response["content-type"] != "application/json":
            raise ValueError("Unexpected contenty type")

        octets = response.body.read()
        dictionary = json.loads(octets)

        LOG.debug("APIStateTracker: received JSON: " +
            json.dumps(dictionary, ensure_ascii=True))

        if not "events" in dictionary:
            return

        if not "current" in dictionary:
            raise ValueError("Incomplete dictionary")

        t = dictionary["t"]
        if not type(t) == types.IntType and not type(t) == types.LongType:
            raise ValueError("Invalid type for current event time")
        if t < 0:
            raise ValueError("Invalid value for current event time")

        self.timestamp = t
        self.process_dictionary(dictionary)
Example #23
0
 def _update_queue(self, lost_stream, ignored):
     ''' Invoked when a connection is lost '''
     queue, found = collections.deque(), False
     position = 0
     for stream in self.queue:
         if not found:
             if lost_stream != stream:
                 position += 1
                 queue.append(stream)
             else:
                 found = True
                 self.known.remove(stream)
         elif not stream.opaque:
             position += 1
             queue.append(stream)
         else:
             request, stream.opaque = stream.opaque, None
             try:
                 self._do_negotiate((stream, request, position))
                 position += 1
                 queue.append(stream)
             except (KeyboardInterrupt, SystemExit):
                 raise
             except:
                 LOG.exception()
                 stream.unregister_atclose(self._update_queue)
                 self.known.remove(stream)
                 stream.close()
     self.queue = queue
Example #24
0
    def open_or_die(self):

        ''' Open the database or die '''

        if not GEOIP:
            LOG.error("Missing dependency: GeoIP")
            LOG.info("Please install GeoIP python wrappers, e.g.")
            LOG.info("    sudo apt-get install python-geoip")
            sys.exit(1)

        path = CONFIG.get("rendezvous.geoip_wrapper.country_database",
                          COUNTRY_DATABASE)

        #
        # Detect the common error case, i.e. that the user has
        # not downloaded the database.  If something fancy is
        # going on, let the GeoIP library stacktrace for us.
        #
        if not os.path.exists(path):
            LOG.error("Missing GeoLiteCountry database: %s" % path)
            LOG.info("Please download it from "
                     "<http://www.maxmind.com/app/geolitecountry>.")
            sys.exit(1)

        self.countries = GEOIP.open(path, GEOIP.GEOIP_STANDARD)
Example #25
0
def main(args):

    CONFIG.register_descriptions({
        "speedtest.client.uri": "Base URI to connect to",
        "speedtest.client.nconn": "Number of concurrent connections to use",
        "speedtest.client.latency_tries": "Number of latency measurements",
    })

    common.main("speedtest.client", "Speedtest client", args)
    conf = CONFIG.copy()

    #
    # If possible use the runner, which will execute the
    # test in the context of the neubot daemon.  Then exit
    # to bypass the POLLER.loop() invokation that is below
    # here.
    # If the runner fails, fallback to the usual code path,
    # which executes the test in the context of the local
    # process.
    # Set 'runned.enabled' to 0 to bypass the runner and
    # run the test locally.
    #
    if (utils.intify(conf['runner.enabled']) and
        runner_clnt.runner_client(conf["agent.api.address"],
                                  conf["agent.api.port"],
                                  LOG.noisy, "speedtest")):
        sys.exit(0)

    LOG.info('Will run the test in the local context...')

    client = ClientSpeedtest(POLLER)
    client.configure(conf)
    client.connect_uri()
    POLLER.loop()
Example #26
0
def main(args):

    daemonize = True
    blink = False
    nohide = True

    try:
        options, arguments = getopt.getopt(args[1:], "BdnqVv", ["help"])
    except getopt.GetoptError:
        sys.stderr.write(USAGE % args[0])
        sys.exit(1)

    for name, value in options:
        if name == "-B":
            blink = True
        elif name == "-d":
            daemonize = False
        elif name == "--help":
            sys.stdout.write(HELP % args[0])
            sys.exit(0)
        elif name == "-n":
            nohide = True
        elif name == "-q":
            nohide = False
        elif name == "-V":
            sys.stderr.write(VERSION + "\n")
            sys.exit(0)
        elif name == "-v":
            LOG.verbose()

    if len(arguments) >= 3:
        sys.stderr.write(USAGE % args[0])
        sys.exit(1)
    elif len(arguments) == 2:
        address = arguments[0]
        port = arguments[1]
    elif len(arguments) == 1:
        address = ADDRESS
        port = arguments[0]
    else:
        address = ADDRESS
        port = PORT

    if daemonize:
        system.change_dir()
        system.go_background()
        LOG.redirect()
    system.drop_privileges(LOG.error)

    gtk.gdk.threads_init()
    icon = StatusIcon(address, port, blink, nohide)
    tracker = StateTrackerThread(icon, address, port)
    tracker.daemon = True
    tracker.start()

    gtk.gdk.threads_enter()
    gtk.main()
    gtk.gdk.threads_leave()
    tracker.interrupt()
Example #27
0
 def connection_lost(self, stream):
     if NOTIFIER.is_subscribed("testdone"):
         LOG.debug("RendezVous: don't _schedule(): test in progress")
         return
     if self._task:
         LOG.debug("RendezVous: don't _schedule(): we already have a task")
         return
     self._schedule()
Example #28
0
 def connect_uri(self, uri=None, count=None):
     if not uri:
         uri = self.conf.get("speedtest.client.uri",
           "http://master.neubot.org/")
     if not count:
         count = self.conf.get("speedtest.client.nconn", 1)
     LOG.info("* speedtest with %s" % uri)
     ClientHTTP.connect_uri(self, uri, count)
Example #29
0
 def connection_lost(self, stream):
     if runner_core.test_is_running():
         LOG.debug("RendezVous: don't _schedule(): test in progress")
         return
     if self._task:
         LOG.debug("RendezVous: don't _schedule(): we already have a task")
         return
     self._schedule()
Example #30
0
 def connection_made(self, sock, rtt=0):
     ''' Invoked when the connection is created '''
     if rtt:
         LOG.debug("ClientHTTP: latency: %s" % utils.time_formatter(rtt))
         self.rtt = rtt
     stream = ClientStream(self.poller)
     stream.attach(self, sock, self.conf)
     self.connection_ready(stream)