Example #1
0
def main(args):
    ''' Main function '''

    try:
        options, arguments = getopt.getopt(args[1:], 'f:n')
    except getopt.error:
        sys.exit('Usage: %s [-n] [-f database] test [negotiate_uri]' % args[0])
    if len(arguments) != 1 and len(arguments) != 2:
        sys.exit('Usage: %s [-n] [-f database] test [negotiate_uri]' % args[0])

    database_path = system.get_default_database_path()
    auto_rendezvous = True
    for name, value in options:
        if name == '-f':
            database_path = value
        elif name == '-n':
            auto_rendezvous = False

    DATABASE.set_path(database_path)
    CONFIG.merge_database(DATABASE.connection())

    if len(arguments) == 2:
        RUNNER_TESTS.update({arguments[0]: [arguments[1]]})
        ctx = {'uri': arguments[1]}
    else:
        ctx = None

    RUNNER_CORE.run(arguments[0], lambda *args: None, auto_rendezvous, ctx)
    POLLER.loop()
Example #2
0
 def _api_exit(self, stream, request, query):
     POLLER.sched(0, POLLER.break_loop)
     response = Message()
     stringio = StringIO.StringIO("See you, space cowboy\n")
     response.compose(code="200", reason="Ok", body=stringio,
                      mimetype="text/plain", keepalive=False)
     stream.send_response(request, response)
Example #3
0
def main(args):

    CONFIG.register_descriptions({
        "speedtest.negotiate.address": "Address to listen to",
        "speedtest.negotiate.auth_only": "Enable doing tests for authorized clients only",
        "speedtest.negotiate.daemonize": "Enable going in background",
        "speedtest.negotiate.port": "Port to listen to",
    })

    common.main("speedtest.negotiate", "Speedtest negotiation server", args)

    conf = CONFIG.copy()

    server = ServerSpeedtest(POLLER)
    server.configure(conf)
    server.listen((conf["speedtest.negotiate.address"],
                  conf["speedtest.negotiate.port"]))

    if conf["speedtest.negotiate.daemonize"]:
        system.change_dir()
        system.go_background()
        LOG.redirect()

    system.drop_privileges(LOG.error)
    POLLER.loop()
Example #4
0
def main(args):

    ''' main() of this module '''

    CONFIG.register_descriptions({
        "http.client.class": "Specify alternate ClientHTTP-like class",
        "http.client.method": "Specify alternate HTTP method",
        "http.client.stdout": "Enable writing response to stdout",
        "http.client.uri": "Specify URI to download from/upload to",
    })

    common.main("http.client", "Simple Neubot HTTP client", args)
    conf = CONFIG.copy()

    make_client = TestClient
    if conf["http.client.class"]:
        make_client = utils.import_class(conf["http.client.class"])

    if not conf["http.client.uri"]:
        sys.stdout.write("Please, specify URI via -D http.client.uri=URI\n")
        sys.exit(0)

    client = make_client(POLLER)
    client.configure(conf)
    client.connect_uri(conf["http.client.uri"])

    POLLER.loop()
    sys.exit(0)
Example #5
0
def main(args):

    CONFIG.register_descriptions({
        "rendezvous.server.address": "Set rendezvous server address",
        "rendezvous.server.daemonize": "Enable daemon behavior",
        "rendezvous.server.ports": "List of rendezvous server ports",
        "rendezvous.server.update_uri": "Where to download updates from",
        "rendezvous.server.update_version": "Update Neubot version number",
        "rendezvous.geoip_wrapper.country_database": "Path of the GeoIP country database",
        "rendezvous.server.default": "Default test server to use",
    })

    common.main("rendezvous.server", "Rendezvous server", args)
    conf = CONFIG.copy()

    HTTP_SERVER.configure(conf)
    for port in conf["rendezvous.server.ports"].split(","):
        HTTP_SERVER.listen((conf["rendezvous.server.address"], int(port)))

    # Really start this module
    run(POLLER, conf)

    if conf["rendezvous.server.daemonize"]:
        system.change_dir()
        system.go_background()
        LOG.redirect()

    system.drop_privileges(LOG.error)
    POLLER.loop()
Example #6
0
def main(args):

    ''' main() function of this module '''

    CONFIG.register_descriptions({
        "http.server.address": "Address to listen to",
        "http.server.class": "Use alternate ServerHTTP-like class",
        "http.server.mime": "Enable code that guess mime types",
        "http.server.ports": "List of ports to listen to",
        "http.server.rootdir": "Root directory for static pages",
        "http.server.ssi": "Enable server-side includes",
    })

    common.main("http.server", "Neubot simple HTTP server", args)
    conf = CONFIG.copy()

    HTTP_SERVER.configure(conf)

    if conf["http.server.rootdir"] == ".":
        conf["http.server.rootdir"] = os.path.abspath(".")

    for port in conf["http.server.ports"].split(","):
        if port:
            HTTP_SERVER.listen((conf["http.server.address"], int(port)))

    POLLER.loop()
Example #7
0
    def test_done(self, *baton):
        ''' Invoked when the test is done '''

        #
        # Stop streaming test events to interested parties
        # via the log streaming API.
        # Do not stop processing immediately and give HTTP
        # code time to stream logs to the client in case
        # connections fails immediately.
        # This must not be done when we're processing the
        # somewhat internal 'rendezvous' or 'mlab-ns' tests.
        #
        if self.queue[0][0] != 'rendezvous' and self.queue[0][0] != 'mlab-ns':
            POLLER.sched(2, STREAMING_LOG.stop_streaming)

        # Paranoid
        if baton[0] != 'testdone':
            raise RuntimeError('runner_core: invoked for the wrong event')

        # Notify the caller that the test is done
        deferred, ctx = self.queue.popleft()[1:]
        deferred.callback(ctx)

        #
        # Allow for more tests
        # If callback() adds one more test, that would
        # be run by the run_queue() invocation below.
        #
        self.running = False

        # Eventually run next queued test
        self.run_queue()
Example #8
0
def main(args):

    CONFIG.register_descriptions({
        "speedtest.client.uri": "Base URI to connect to",
        "speedtest.client.nconn": "Number of concurrent connections to use",
        "speedtest.client.latency_tries": "Number of latency measurements",
    })

    common.main("speedtest.client", "Speedtest client", args)
    conf = CONFIG.copy()

    #
    # If possible use the runner, which will execute the
    # test in the context of the neubot daemon.  Then exit
    # to bypass the POLLER.loop() invokation that is below
    # here.
    # If the runner fails, fallback to the usual code path,
    # which executes the test in the context of the local
    # process.
    # Set 'runned.enabled' to 0 to bypass the runner and
    # run the test locally.
    #
    if (utils.intify(conf['runner.enabled']) and
        runner_clnt.runner_client(conf["agent.api.address"],
                                  conf["agent.api.port"],
                                  LOG.noisy, "speedtest")):
        sys.exit(0)

    LOG.info('Will run the test in the local context...')

    client = ClientSpeedtest(POLLER)
    client.configure(conf)
    client.connect_uri()
    POLLER.loop()
Example #9
0
def main(args):
    ''' Main function '''

    try:
        options, arguments = getopt.getopt(args[1:], 'f:nv')
    except getopt.error:
        sys.exit(USAGE)

    database_path = system.get_default_database_path()
    auto_discover = True
    for name, value in options:
        if name == '-f':
            database_path = value
        elif name == '-n':
            auto_discover = False
        elif name == '-v':
            CONFIG['verbose'] = 1

    if len(arguments) != 1 and len(arguments) != 2:
        sys.exit(USAGE)

    DATABASE.set_path(database_path)
    CONFIG.merge_database(DATABASE.connection())

    if len(arguments) == 2:
        RUNNER_TESTS.update({arguments[0]: [arguments[1]]})
        ctx = {'uri': arguments[1]}
    else:
        ctx = None

    deferred = Deferred()
    deferred.add_callback(lambda param: None)
    RUNNER_CORE.run(arguments[0], deferred, auto_discover, ctx)
    POLLER.loop()
Example #10
0
def main(args):

    ''' Main function '''

    common.main("rendezvous.client", "Rendezvous client", args)
    client = ClientRendezvous()
    client.run()
    POLLER.loop()
Example #11
0
def main(args):
    """ Main function """

    if not system.has_enough_privs():
        sys.exit('FATAL: you must be root')

    common.main("agent", "Run in background, periodically run tests", args)

    conf = CONFIG.copy()

    privacy.complain_if_needed()

    BACKEND.use_backend("neubot")
    BACKEND.datadir_init()

    # FIXME We're ignoring agent.api.{address,port} that are now
    # deprecated and should be removed soon.
    background_api.start_api()

    if conf["agent.daemonize"]:
        LOG.redirect()
        system.go_background()

    if conf["agent.use_syslog"]:
        LOG.redirect()

    #
    # When we run as an agent we also save logs into
    # the database, to easily access and show them via
    # the web user interface.
    #
    LOG.use_database()

    logging.info('%s for POSIX: starting up', utils_version.PRODUCT)

    system.drop_privileges()

    if os.getuid() == 0 or os.geteuid() == 0:
        logging.error('agent: still running as root')
        os._exit(1)

    if conf["agent.rendezvous"]:
        BACKGROUND_RENDEZVOUS.start()

    POLLER.loop()

    logging.info('%s for POSIX: shutting down', utils_version.PRODUCT)
    LOG.writeback()

    #
    # Make sure that we do not leave the database
    # in an inconsistent state.
    #
    DATABASE.close()
Example #12
0
def main(args):

    CONFIG.register_descriptions({
        # General variables
        "net.stream.certfile": "Set SSL certfile path",
        "net.stream.ipv6": "Enable IPv6",
        "net.stream.key": "Set key for ARC4",
        "net.stream.secure": "Enable SSL",
        "net.stream.server_side": "Enable SSL server-side mode",
        "net.stream.rcvbuf": "Set sock recv buffer (0 = use default)",
        "net.stream.sndbuf": "Set sock send buffer (0 = use default)",
        # For main()
        "net.stream.address": "Set client or server address",
        "net.stream.chunk": "Chunk written by each write",
        "net.stream.clients": "Set number of client connections",
        "net.stream.daemonize": "Enable daemon behavior",
        "net.stream.duration": "Set duration of a test",
        "net.stream.listen": "Enable server mode",
        "net.stream.port": "Set client or server port",
        "net.stream.proto": "Set proto (chargen, discard, or echo)",
    })

    common.main("net.stream", "TCP bulk transfer test", args)

    conf = CONFIG.copy()

    endpoint = (conf["net.stream.address"], conf["net.stream.port"])

    if not conf["net.stream.proto"]:
        if conf["net.stream.listen"]:
            conf["net.stream.proto"] = "chargen"
        else:
            conf["net.stream.proto"] = "discard"
    elif conf["net.stream.proto"] not in ("chargen", "discard", "echo"):
        common.write_help(sys.stderr, "net.stream", "TCP bulk transfer test")
        sys.exit(1)

    handler = GenericHandler(POLLER)
    handler.configure(conf)

    if conf["net.stream.listen"]:
        if conf["net.stream.daemonize"]:
            system.change_dir()
            system.go_background()
            LOG.redirect()
        system.drop_privileges(LOG.error)
        conf["net.stream.server_side"] = True
        handler.listen(endpoint)
    else:
        handler.connect(endpoint, count=conf["net.stream.clients"])

    POLLER.loop()
    sys.exit(0)
Example #13
0
 def _api_debug(self, stream, request, query):
     response = Message()
     debuginfo = {}
     NOTIFIER.snap(debuginfo)
     POLLER.snap(debuginfo)
     debuginfo["queue_history"] = QUEUE_HISTORY
     debuginfo["WWW"] = WWW
     stringio = StringIO.StringIO()
     pprint.pprint(debuginfo, stringio)
     stringio.seek(0)
     response.compose(code="200", reason="Ok", body=stringio,
                      mimetype="text/plain")
     stream.send_response(request, response)
Example #14
0
 def connection_made(self):
     self.buffer = "A" * self.conf["net.stream.chunk"]
     duration = self.conf["net.stream.duration"]
     if duration >= 0:
         POLLER.sched(duration, self._do_close)
     if self.kind == "discard":
         self.start_recv()
     elif self.kind == "chargen":
         self.start_send(self.buffer)
     elif self.kind == "echo":
         self.start_recv()
     else:
         self.close()
Example #15
0
 def _api_debug(stream, request, query):
     ''' Implements /api/debug URI '''
     response = Message()
     debuginfo = {}
     NOTIFIER.snap(debuginfo)
     POLLER.snap(debuginfo)
     debuginfo["queue_history"] = QUEUE_HISTORY
     debuginfo["WWWDIR"] = utils_hier.WWWDIR
     gc.collect()
     debuginfo['typestats'] = objgraph.typestats()
     body = pprint.pformat(debuginfo)
     response.compose(code="200", reason="Ok", body=body,
                      mimetype="text/plain")
     stream.send_response(request, response)
Example #16
0
def main(args):
    """ Main function """
    try:
        options, arguments = getopt.getopt(args[1:], "6A:fp:v")
    except getopt.error:
        sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]")
    if arguments:
        sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]")

    prefer_ipv6 = 0
    address = "master.neubot.org"
    force = 0
    port = 8080
    noisy = 0
    for name, value in options:
        if name == "-6":
            prefer_ipv6 = 1
        elif name == "-A":
            address = value
        elif name == "-f":
            force = 1
        elif name == "-p":
            port = int(value)
        elif name == "-v":
            noisy = 1

    if os.path.isfile(DATABASE.path):
        DATABASE.connect()
        CONFIG.merge_database(DATABASE.connection())
    else:
        logging.warning("speedtest: database file is missing: %s", DATABASE.path)
        BACKEND.use_backend("null")
    if noisy:
        log.set_verbose()

    conf = CONFIG.copy()
    conf["speedtest.client.uri"] = "http://%s:%d/" % (address, port)
    conf["prefer_ipv6"] = prefer_ipv6

    if not force:
        if runner_clnt.runner_client(conf["agent.api.address"], conf["agent.api.port"], CONFIG["verbose"], "speedtest"):
            sys.exit(0)
        logging.warning("speedtest: failed to contact Neubot; is Neubot running?")
        sys.exit(1)

    logging.info("speedtest: run the test in the local process context...")
    client = ClientSpeedtest(POLLER)
    client.configure(conf)
    client.connect_uri()
    POLLER.loop()
Example #17
0
 def connection_made(self):
     self.buffer = "A" * self.conf.get("net.stream.chunk", 32768)
     MEASURER.register_stream(self)
     duration = self.conf.get("net.stream.duration", 10)
     if duration >= 0:
         POLLER.sched(duration, self._do_close)
     if self.kind == "discard":
         self.start_recv()
     elif self.kind == "chargen":
         self.start_send(self.buffer)
     elif self.kind == "echo":
         self.start_recv()
     else:
         self.close()
Example #18
0
def main(args):

    CONFIG.register_descriptions({
        "speedtest.client.uri": "Base URI to connect to",
        "speedtest.client.nconn": "Number of concurrent connections to use",
        "speedtest.client.latency_tries": "Number of latency measurements",
    })

    common.main("speedtest.client", "Speedtest client", args)
    conf = CONFIG.copy()
    client = ClientSpeedtest(POLLER)
    client.configure(conf)
    client.connect_uri()
    POLLER.loop()
Example #19
0
def main(args):

    CONFIG.register_descriptions({
        "rendezvous.client.debug": "Do not perform any test",
        "rendezvous.client.version": "Set rendezvous client version",
    })

    common.main("rendezvous.client", "Rendezvous client", args)
    conf = CONFIG.copy()

    client = ClientRendezvous(POLLER)
    client.configure(conf)
    client.connect_uri()

    POLLER.loop()
Example #20
0
def main(args):
    common.main("show_database", "Show a database in the Web GUI", args)

    HTTP_SERVER.configure({
                           "http.server.rootdir": WWW,
                           "http.server.ssi": True,
                           "http.server.bind_or_die": True
                          })

    HTTP_SERVER.register_child(ServerAPI(POLLER), "/api")
    HTTP_SERVER.listen(("127.0.0.1", 8080))

    browser.open_patient("127.0.0.1", "8080", True)

    POLLER.loop()
Example #21
0
    def _periodic(self, *args, **kwargs):
        POLLER.sched(INTERVAL, self._periodic)
        self._tofire, q = [], self._tofire
        for event in q:

            #
            # WARNING! Please resist the temptation of merging
            # the [] and the del into a single pop() because that
            # will not work: this is a defaultdict and so here
            # event might not be in _subscribers.
            #
            queue = self._subscribers[event]
            del self._subscribers[event]

            logging.debug("* periodic: %s", event)

            self._fireq(event, queue)
Example #22
0
def main(args):

    config.register_descriptions()
    common.main("bittorrent", "Neubot BitTorrent module", args)
    conf = CONFIG.copy()
    config.finalize_conf(conf)

    if conf["bittorrent.listen"]:

        #
        # If we need to negotiate and we're runing
        # standalone we also need to bring up the
        # global HTTP server.
        #
        if conf["bittorrent.negotiate"]:
            HTTP_SERVER.configure(conf)
            HTTP_SERVER.listen((conf["bittorrent.address"],
                               conf["bittorrent.negotiate.port"]))
            conf["negotiate.listen"] = True
            negotiate.run(POLLER, conf)

        #
        # Drop privileges after listen() so we can
        # bind() to privileged ports
        #
        if conf["bittorrent.daemonize"]:
            system.change_dir()
            system.go_background()
            LOG.redirect()

        system.drop_privileges(LOG.error)

    else:
        #
        # When we're connecting to a remote host to perform a test
        # we want Neubot to quit at the end of the test.  When this
        # happens the test code publishes the "testdone" event, so
        # here we prepare to intercept the event and break our main
        # loop.
        #
        NOTIFIER.subscribe("testdone", lambda event, ctx: POLLER.break_loop())

    run(POLLER, conf)

    POLLER.loop()
Example #23
0
def main(args):

    common.main("speedtest.server", "Speedtest Test Server", args)

    conf = CONFIG.copy()

    server = ServerTest(POLLER)
    server.configure(conf)
    server.listen((conf["speedtest.server.address"],
                  conf["speedtest.server.port"]))

    if conf["speedtest.server.daemonize"]:
        system.change_dir()
        system.go_background()
        LOG.redirect()

    system.drop_privileges(LOG.error)
    POLLER.loop()
Example #24
0
def main(args):

    CONFIG.register_descriptions({
        # General variables
        "net.stream.certfile": "Set SSL certfile path",
        "net.stream.secure": "Enable SSL",
        "net.stream.server_side": "Enable SSL server-side mode",
        # For main()
        "net.stream.address": "Set client or server address",
        "net.stream.chunk": "Chunk written by each write",
        "net.stream.clients": "Set number of client connections",
        "net.stream.duration": "Set duration of a test",
        "net.stream.listen": "Enable server mode",
        "net.stream.port": "Set client or server port",
        "net.stream.proto": "Set proto (chargen, discard, or echo)",
    })

    common.main("net.stream", "TCP bulk transfer test", args)

    conf = CONFIG.copy()

    endpoint = (conf["net.stream.address"], conf["net.stream.port"])

    if not conf["net.stream.proto"]:
        if conf["net.stream.listen"]:
            conf["net.stream.proto"] = "chargen"
        else:
            conf["net.stream.proto"] = "discard"
    elif conf["net.stream.proto"] not in ("chargen", "discard", "echo"):
        common.write_help(sys.stderr, "net.stream", "TCP bulk transfer test")
        sys.exit(1)

    handler = GenericHandler(POLLER)
    handler.configure(conf)

    if conf["net.stream.listen"]:
        conf["net.stream.server_side"] = True
        handler.listen(endpoint)
    else:
        handler.connect(endpoint, count=conf["net.stream.clients"])

    POLLER.loop()
    sys.exit(0)
Example #25
0
    def test_done(self, *baton):
        ''' Invoked when the test is done '''

        #
        # Stop streaming test events to interested parties
        # via the log streaming API.
        # Do not stop processing immediately and give HTTP
        # code time to stream logs to the client in case
        # connections fails immediately.
        # This must not be done when we're processing the
        # somewhat internal 'rendezvous' test.
        #
        if self.queue[0][0] != 'rendezvous':
            POLLER.sched(2, LOG.stop_streaming)

        # Paranoid
        if baton[0] != 'testdone':
            raise RuntimeError('Invoked for the wrong event')

        # Notify the caller that the test is done
        callback, ctx = self.queue.popleft()[1:]
        try:
            if ctx:
                callback(ctx)
            else:
                callback()
        except (KeyboardInterrupt, SystemExit):
            raise
        except:
            exc = asyncore.compact_traceback()
            error = str(exc)
            LOG.error('runner_core: catched exception: %s' % error)

        #
        # Allow for more tests
        # If callback() adds one more test, that would
        # be run by the run_queue() invocation below.
        #
        self.running = False

        # Eventually run next queued test
        self.run_queue()
Example #26
0
def main(args):
    ''' Main function '''

    try:
        options, arguments = getopt.getopt(args[1:], 'f')
    except getopt.error:
        sys.exit('Usage: %s [-f database] test negotiate_uri' % args[0])
    if len(arguments) != 2:
        sys.exit('Usage: %s [-f database] test negotiate_uri' % args[0])

    database_path = system.get_default_database_path()
    for name, value in options:
        if name == '-f':
            database_path = value

    DATABASE.set_path(database_path)
    CONFIG.merge_database(DATABASE.connection())

    run(arguments[0], arguments[1], lambda: None)
    POLLER.loop()
Example #27
0
    def _do_update_queue(self):

        pos = 1
        for session in self.queue:
            if pos <= 3 and not session.active:
                session.active = True
            session.queuepos = pos
            pos = pos + 1

        if not self.task:
            self.task = POLLER.sched(60, self._sample_queue_length)
Example #28
0
def main(args):

    ''' main() function of this module '''

    CONFIG.register_descriptions({
        "http.server.address": "Address to listen to",
        "http.server.class": "Use alternate ServerHTTP-like class",
        "http.server.daemonize": "Run in background as a daemon",
        "http.server.mime": "Enable code that guess mime types",
        "http.server.ports": "List of ports to listen to",
        "http.server.rootdir": "Root directory for static pages",
        "http.server.ssi": "Enable server-side includes",
    })

    common.main("http.server", "Neubot simple HTTP server", args)
    conf = CONFIG.copy()

    if conf["http.server.class"]:
        make_child = utils.import_class(conf["http.server.class"])
        server = make_child(POLLER)
    else:
        server = HTTP_SERVER

    server.configure(conf)

    if conf["http.server.rootdir"] == ".":
        conf["http.server.rootdir"] = os.path.abspath(".")

    for port in conf["http.server.ports"].split(","):
        if port:
            server.listen((conf["http.server.address"], int(port)))

    if conf["http.server.daemonize"]:
        system.change_dir()
        system.go_background()
        system.write_pidfile()
        LOG.redirect()

    system.drop_privileges(LOG.error)

    POLLER.loop()
Example #29
0
def main(args):
    common.main("agent", "Run in background, periodically run tests", args)

    conf = CONFIG.copy()

    privacy.complain_if_needed()

    if conf["agent.api"]:
        server = HTTP_SERVER
        LOG.debug("* API server root directory: %s" % WWW)
        conf["http.server.rootdir"] = WWW
        conf["http.server.ssi"] = True
        conf["http.server.bind_or_die"] = True
        server.configure(conf)
        server.register_child(ServerAPI(POLLER), "/api")
        server.listen((conf["agent.api.address"], conf["agent.api.port"]))

    if conf["agent.daemonize"]:
        system.change_dir()
        system.go_background()
        system.write_pidfile()
        LOG.redirect()

    if conf["agent.use_syslog"]:
        LOG.redirect()

    system.drop_privileges(LOG.error)

    #
    # When we run as an agent we also save logs into
    # the database, to easily access and show them via
    # the web user interface.
    #
    LOG.use_database()

    if conf["agent.rendezvous"]:
        client = ClientRendezvous(POLLER)
        client.configure(conf)
        client.connect_uri()

    POLLER.loop()
Example #30
0
def main():
    Sflag = False

    options, arguments = getopt.getopt(sys.argv[1:], "S")
    for key, value in options:
        if key == "-S":
            Sflag = True

    # don't clobber my local database
    DATABASE.set_path(":memory:")
    DATABASE.connect()

    if Sflag:
        server = ServerSpeedtest(POLLER)
        server.configure({"speedtest.negotiate.daemonize": False})
        server.listen(("127.0.0.1", 8080))

    else:
        speedtest_again()

    POLLER.loop()