def main(args):
''' Main function '''
try:
options, arguments = getopt.getopt(args[1:], 'f:n')
except getopt.error:
sys.exit('Usage: %s [-n] [-f database] test [negotiate_uri]' % args[0])
if len(arguments) != 1 and len(arguments) != 2:
sys.exit('Usage: %s [-n] [-f database] test [negotiate_uri]' % args[0])
database_path = system.get_default_database_path()
auto_rendezvous = True
for name, value in options:
if name == '-f':
database_path = value
elif name == '-n':
auto_rendezvous = False
DATABASE.set_path(database_path)
CONFIG.merge_database(DATABASE.connection())
if len(arguments) == 2:
RUNNER_TESTS.update({arguments[0]: [arguments[1]]})
ctx = {'uri': arguments[1]}
else:
ctx = None
RUNNER_CORE.run(arguments[0], lambda *args: None, auto_rendezvous, ctx)
POLLER.loop()
def _api_exit(self, stream, request, query):
POLLER.sched(0, POLLER.break_loop)
response = Message()
stringio = StringIO.StringIO("See you, space cowboy\n")
response.compose(code="200", reason="Ok", body=stringio,
mimetype="text/plain", keepalive=False)
stream.send_response(request, response)
def main(args):
CONFIG.register_descriptions({
"speedtest.negotiate.address": "Address to listen to",
"speedtest.negotiate.auth_only": "Enable doing tests for authorized clients only",
"speedtest.negotiate.daemonize": "Enable going in background",
"speedtest.negotiate.port": "Port to listen to",
})
common.main("speedtest.negotiate", "Speedtest negotiation server", args)
conf = CONFIG.copy()
server = ServerSpeedtest(POLLER)
server.configure(conf)
server.listen((conf["speedtest.negotiate.address"],
conf["speedtest.negotiate.port"]))
if conf["speedtest.negotiate.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
''' main() of this module '''
CONFIG.register_descriptions({
"http.client.class": "Specify alternate ClientHTTP-like class",
"http.client.method": "Specify alternate HTTP method",
"http.client.stdout": "Enable writing response to stdout",
"http.client.uri": "Specify URI to download from/upload to",
})
common.main("http.client", "Simple Neubot HTTP client", args)
conf = CONFIG.copy()
make_client = TestClient
if conf["http.client.class"]:
make_client = utils.import_class(conf["http.client.class"])
if not conf["http.client.uri"]:
sys.stdout.write("Please, specify URI via -D http.client.uri=URI\n")
sys.exit(0)
client = make_client(POLLER)
client.configure(conf)
client.connect_uri(conf["http.client.uri"])
POLLER.loop()
sys.exit(0)
def main(args):
CONFIG.register_descriptions({
"rendezvous.server.address": "Set rendezvous server address",
"rendezvous.server.daemonize": "Enable daemon behavior",
"rendezvous.server.ports": "List of rendezvous server ports",
"rendezvous.server.update_uri": "Where to download updates from",
"rendezvous.server.update_version": "Update Neubot version number",
"rendezvous.geoip_wrapper.country_database": "Path of the GeoIP country database",
"rendezvous.server.default": "Default test server to use",
})
common.main("rendezvous.server", "Rendezvous server", args)
conf = CONFIG.copy()
HTTP_SERVER.configure(conf)
for port in conf["rendezvous.server.ports"].split(","):
HTTP_SERVER.listen((conf["rendezvous.server.address"], int(port)))
# Really start this module
run(POLLER, conf)
if conf["rendezvous.server.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
''' main() function of this module '''
CONFIG.register_descriptions({
"http.server.address": "Address to listen to",
"http.server.class": "Use alternate ServerHTTP-like class",
"http.server.mime": "Enable code that guess mime types",
"http.server.ports": "List of ports to listen to",
"http.server.rootdir": "Root directory for static pages",
"http.server.ssi": "Enable server-side includes",
})
common.main("http.server", "Neubot simple HTTP server", args)
conf = CONFIG.copy()
HTTP_SERVER.configure(conf)
if conf["http.server.rootdir"] == ".":
conf["http.server.rootdir"] = os.path.abspath(".")
for port in conf["http.server.ports"].split(","):
if port:
HTTP_SERVER.listen((conf["http.server.address"], int(port)))
POLLER.loop()
def test_done(self, *baton):
''' Invoked when the test is done '''
#
# Stop streaming test events to interested parties
# via the log streaming API.
# Do not stop processing immediately and give HTTP
# code time to stream logs to the client in case
# connections fails immediately.
# This must not be done when we're processing the
# somewhat internal 'rendezvous' or 'mlab-ns' tests.
#
if self.queue[0][0] != 'rendezvous' and self.queue[0][0] != 'mlab-ns':
POLLER.sched(2, STREAMING_LOG.stop_streaming)
# Paranoid
if baton[0] != 'testdone':
raise RuntimeError('runner_core: invoked for the wrong event')
# Notify the caller that the test is done
deferred, ctx = self.queue.popleft()[1:]
deferred.callback(ctx)
#
# Allow for more tests
# If callback() adds one more test, that would
# be run by the run_queue() invocation below.
#
self.running = False
# Eventually run next queued test
self.run_queue()
def main(args):
CONFIG.register_descriptions({
"speedtest.client.uri": "Base URI to connect to",
"speedtest.client.nconn": "Number of concurrent connections to use",
"speedtest.client.latency_tries": "Number of latency measurements",
})
common.main("speedtest.client", "Speedtest client", args)
conf = CONFIG.copy()
#
# If possible use the runner, which will execute the
# test in the context of the neubot daemon. Then exit
# to bypass the POLLER.loop() invokation that is below
# here.
# If the runner fails, fallback to the usual code path,
# which executes the test in the context of the local
# process.
# Set 'runned.enabled' to 0 to bypass the runner and
# run the test locally.
#
if (utils.intify(conf['runner.enabled']) and
runner_clnt.runner_client(conf["agent.api.address"],
conf["agent.api.port"],
LOG.noisy, "speedtest")):
sys.exit(0)
LOG.info('Will run the test in the local context...')
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
''' Main function '''
try:
options, arguments = getopt.getopt(args[1:], 'f:nv')
except getopt.error:
sys.exit(USAGE)
database_path = system.get_default_database_path()
auto_discover = True
for name, value in options:
if name == '-f':
database_path = value
elif name == '-n':
auto_discover = False
elif name == '-v':
CONFIG['verbose'] = 1
if len(arguments) != 1 and len(arguments) != 2:
sys.exit(USAGE)
DATABASE.set_path(database_path)
CONFIG.merge_database(DATABASE.connection())
if len(arguments) == 2:
RUNNER_TESTS.update({arguments[0]: [arguments[1]]})
ctx = {'uri': arguments[1]}
else:
ctx = None
deferred = Deferred()
deferred.add_callback(lambda param: None)
RUNNER_CORE.run(arguments[0], deferred, auto_discover, ctx)
POLLER.loop()
def main(args):
''' Main function '''
common.main("rendezvous.client", "Rendezvous client", args)
client = ClientRendezvous()
client.run()
POLLER.loop()
def main(args):
""" Main function """
if not system.has_enough_privs():
sys.exit('FATAL: you must be root')
common.main("agent", "Run in background, periodically run tests", args)
conf = CONFIG.copy()
privacy.complain_if_needed()
BACKEND.use_backend("neubot")
BACKEND.datadir_init()
# FIXME We're ignoring agent.api.{address,port} that are now
# deprecated and should be removed soon.
background_api.start_api()
if conf["agent.daemonize"]:
LOG.redirect()
system.go_background()
if conf["agent.use_syslog"]:
LOG.redirect()
#
# When we run as an agent we also save logs into
# the database, to easily access and show them via
# the web user interface.
#
LOG.use_database()
logging.info('%s for POSIX: starting up', utils_version.PRODUCT)
system.drop_privileges()
if os.getuid() == 0 or os.geteuid() == 0:
logging.error('agent: still running as root')
os._exit(1)
if conf["agent.rendezvous"]:
BACKGROUND_RENDEZVOUS.start()
POLLER.loop()
logging.info('%s for POSIX: shutting down', utils_version.PRODUCT)
LOG.writeback()
#
# Make sure that we do not leave the database
# in an inconsistent state.
#
DATABASE.close()
def main(args):
CONFIG.register_descriptions({
# General variables
"net.stream.certfile": "Set SSL certfile path",
"net.stream.ipv6": "Enable IPv6",
"net.stream.key": "Set key for ARC4",
"net.stream.secure": "Enable SSL",
"net.stream.server_side": "Enable SSL server-side mode",
"net.stream.rcvbuf": "Set sock recv buffer (0 = use default)",
"net.stream.sndbuf": "Set sock send buffer (0 = use default)",
# For main()
"net.stream.address": "Set client or server address",
"net.stream.chunk": "Chunk written by each write",
"net.stream.clients": "Set number of client connections",
"net.stream.daemonize": "Enable daemon behavior",
"net.stream.duration": "Set duration of a test",
"net.stream.listen": "Enable server mode",
"net.stream.port": "Set client or server port",
"net.stream.proto": "Set proto (chargen, discard, or echo)",
})
common.main("net.stream", "TCP bulk transfer test", args)
conf = CONFIG.copy()
endpoint = (conf["net.stream.address"], conf["net.stream.port"])
if not conf["net.stream.proto"]:
if conf["net.stream.listen"]:
conf["net.stream.proto"] = "chargen"
else:
conf["net.stream.proto"] = "discard"
elif conf["net.stream.proto"] not in ("chargen", "discard", "echo"):
common.write_help(sys.stderr, "net.stream", "TCP bulk transfer test")
sys.exit(1)
handler = GenericHandler(POLLER)
handler.configure(conf)
if conf["net.stream.listen"]:
if conf["net.stream.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
conf["net.stream.server_side"] = True
handler.listen(endpoint)
else:
handler.connect(endpoint, count=conf["net.stream.clients"])
POLLER.loop()
sys.exit(0)
def _api_debug(self, stream, request, query):
response = Message()
debuginfo = {}
NOTIFIER.snap(debuginfo)
POLLER.snap(debuginfo)
debuginfo["queue_history"] = QUEUE_HISTORY
debuginfo["WWW"] = WWW
stringio = StringIO.StringIO()
pprint.pprint(debuginfo, stringio)
stringio.seek(0)
response.compose(code="200", reason="Ok", body=stringio,
mimetype="text/plain")
stream.send_response(request, response)
def connection_made(self):
self.buffer = "A" * self.conf["net.stream.chunk"]
duration = self.conf["net.stream.duration"]
if duration >= 0:
POLLER.sched(duration, self._do_close)
if self.kind == "discard":
self.start_recv()
elif self.kind == "chargen":
self.start_send(self.buffer)
elif self.kind == "echo":
self.start_recv()
else:
self.close()
def _api_debug(stream, request, query):
''' Implements /api/debug URI '''
response = Message()
debuginfo = {}
NOTIFIER.snap(debuginfo)
POLLER.snap(debuginfo)
debuginfo["queue_history"] = QUEUE_HISTORY
debuginfo["WWWDIR"] = utils_hier.WWWDIR
gc.collect()
debuginfo['typestats'] = objgraph.typestats()
body = pprint.pformat(debuginfo)
response.compose(code="200", reason="Ok", body=body,
mimetype="text/plain")
stream.send_response(request, response)
def main(args):
""" Main function """
try:
options, arguments = getopt.getopt(args[1:], "6A:fp:v")
except getopt.error:
sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]")
if arguments:
sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]")
prefer_ipv6 = 0
address = "master.neubot.org"
force = 0
port = 8080
noisy = 0
for name, value in options:
if name == "-6":
prefer_ipv6 = 1
elif name == "-A":
address = value
elif name == "-f":
force = 1
elif name == "-p":
port = int(value)
elif name == "-v":
noisy = 1
if os.path.isfile(DATABASE.path):
DATABASE.connect()
CONFIG.merge_database(DATABASE.connection())
else:
logging.warning("speedtest: database file is missing: %s", DATABASE.path)
BACKEND.use_backend("null")
if noisy:
log.set_verbose()
conf = CONFIG.copy()
conf["speedtest.client.uri"] = "http://%s:%d/" % (address, port)
conf["prefer_ipv6"] = prefer_ipv6
if not force:
if runner_clnt.runner_client(conf["agent.api.address"], conf["agent.api.port"], CONFIG["verbose"], "speedtest"):
sys.exit(0)
logging.warning("speedtest: failed to contact Neubot; is Neubot running?")
sys.exit(1)
logging.info("speedtest: run the test in the local process context...")
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def connection_made(self):
self.buffer = "A" * self.conf.get("net.stream.chunk", 32768)
MEASURER.register_stream(self)
duration = self.conf.get("net.stream.duration", 10)
if duration >= 0:
POLLER.sched(duration, self._do_close)
if self.kind == "discard":
self.start_recv()
elif self.kind == "chargen":
self.start_send(self.buffer)
elif self.kind == "echo":
self.start_recv()
else:
self.close()
def main(args):
CONFIG.register_descriptions({
"speedtest.client.uri": "Base URI to connect to",
"speedtest.client.nconn": "Number of concurrent connections to use",
"speedtest.client.latency_tries": "Number of latency measurements",
})
common.main("speedtest.client", "Speedtest client", args)
conf = CONFIG.copy()
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
CONFIG.register_descriptions({
"rendezvous.client.debug": "Do not perform any test",
"rendezvous.client.version": "Set rendezvous client version",
})
common.main("rendezvous.client", "Rendezvous client", args)
conf = CONFIG.copy()
client = ClientRendezvous(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
common.main("show_database", "Show a database in the Web GUI", args)
HTTP_SERVER.configure({
"http.server.rootdir": WWW,
"http.server.ssi": True,
"http.server.bind_or_die": True
})
HTTP_SERVER.register_child(ServerAPI(POLLER), "/api")
HTTP_SERVER.listen(("127.0.0.1", 8080))
browser.open_patient("127.0.0.1", "8080", True)
POLLER.loop()
def _periodic(self, *args, **kwargs):
POLLER.sched(INTERVAL, self._periodic)
self._tofire, q = [], self._tofire
for event in q:
#
# WARNING! Please resist the temptation of merging
# the [] and the del into a single pop() because that
# will not work: this is a defaultdict and so here
# event might not be in _subscribers.
#
queue = self._subscribers[event]
del self._subscribers[event]
logging.debug("* periodic: %s", event)
self._fireq(event, queue)
def main(args):
config.register_descriptions()
common.main("bittorrent", "Neubot BitTorrent module", args)
conf = CONFIG.copy()
config.finalize_conf(conf)
if conf["bittorrent.listen"]:
#
# If we need to negotiate and we're runing
# standalone we also need to bring up the
# global HTTP server.
#
if conf["bittorrent.negotiate"]:
HTTP_SERVER.configure(conf)
HTTP_SERVER.listen((conf["bittorrent.address"],
conf["bittorrent.negotiate.port"]))
conf["negotiate.listen"] = True
negotiate.run(POLLER, conf)
#
# Drop privileges after listen() so we can
# bind() to privileged ports
#
if conf["bittorrent.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
else:
#
# When we're connecting to a remote host to perform a test
# we want Neubot to quit at the end of the test. When this
# happens the test code publishes the "testdone" event, so
# here we prepare to intercept the event and break our main
# loop.
#
NOTIFIER.subscribe("testdone", lambda event, ctx: POLLER.break_loop())
run(POLLER, conf)
POLLER.loop()
def main(args):
common.main("speedtest.server", "Speedtest Test Server", args)
conf = CONFIG.copy()
server = ServerTest(POLLER)
server.configure(conf)
server.listen((conf["speedtest.server.address"],
conf["speedtest.server.port"]))
if conf["speedtest.server.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
CONFIG.register_descriptions({
# General variables
"net.stream.certfile": "Set SSL certfile path",
"net.stream.secure": "Enable SSL",
"net.stream.server_side": "Enable SSL server-side mode",
# For main()
"net.stream.address": "Set client or server address",
"net.stream.chunk": "Chunk written by each write",
"net.stream.clients": "Set number of client connections",
"net.stream.duration": "Set duration of a test",
"net.stream.listen": "Enable server mode",
"net.stream.port": "Set client or server port",
"net.stream.proto": "Set proto (chargen, discard, or echo)",
})
common.main("net.stream", "TCP bulk transfer test", args)
conf = CONFIG.copy()
endpoint = (conf["net.stream.address"], conf["net.stream.port"])
if not conf["net.stream.proto"]:
if conf["net.stream.listen"]:
conf["net.stream.proto"] = "chargen"
else:
conf["net.stream.proto"] = "discard"
elif conf["net.stream.proto"] not in ("chargen", "discard", "echo"):
common.write_help(sys.stderr, "net.stream", "TCP bulk transfer test")
sys.exit(1)
handler = GenericHandler(POLLER)
handler.configure(conf)
if conf["net.stream.listen"]:
conf["net.stream.server_side"] = True
handler.listen(endpoint)
else:
handler.connect(endpoint, count=conf["net.stream.clients"])
POLLER.loop()
sys.exit(0)
def test_done(self, *baton):
''' Invoked when the test is done '''
#
# Stop streaming test events to interested parties
# via the log streaming API.
# Do not stop processing immediately and give HTTP
# code time to stream logs to the client in case
# connections fails immediately.
# This must not be done when we're processing the
# somewhat internal 'rendezvous' test.
#
if self.queue[0][0] != 'rendezvous':
POLLER.sched(2, LOG.stop_streaming)
# Paranoid
if baton[0] != 'testdone':
raise RuntimeError('Invoked for the wrong event')
# Notify the caller that the test is done
callback, ctx = self.queue.popleft()[1:]
try:
if ctx:
callback(ctx)
else:
callback()
except (KeyboardInterrupt, SystemExit):
raise
except:
exc = asyncore.compact_traceback()
error = str(exc)
LOG.error('runner_core: catched exception: %s' % error)
#
# Allow for more tests
# If callback() adds one more test, that would
# be run by the run_queue() invocation below.
#
self.running = False
# Eventually run next queued test
self.run_queue()
def main(args):
''' Main function '''
try:
options, arguments = getopt.getopt(args[1:], 'f')
except getopt.error:
sys.exit('Usage: %s [-f database] test negotiate_uri' % args[0])
if len(arguments) != 2:
sys.exit('Usage: %s [-f database] test negotiate_uri' % args[0])
database_path = system.get_default_database_path()
for name, value in options:
if name == '-f':
database_path = value
DATABASE.set_path(database_path)
CONFIG.merge_database(DATABASE.connection())
run(arguments[0], arguments[1], lambda: None)
POLLER.loop()
def _do_update_queue(self):
pos = 1
for session in self.queue:
if pos <= 3 and not session.active:
session.active = True
session.queuepos = pos
pos = pos + 1
if not self.task:
self.task = POLLER.sched(60, self._sample_queue_length)
def main(args):
''' main() function of this module '''
CONFIG.register_descriptions({
"http.server.address": "Address to listen to",
"http.server.class": "Use alternate ServerHTTP-like class",
"http.server.daemonize": "Run in background as a daemon",
"http.server.mime": "Enable code that guess mime types",
"http.server.ports": "List of ports to listen to",
"http.server.rootdir": "Root directory for static pages",
"http.server.ssi": "Enable server-side includes",
})
common.main("http.server", "Neubot simple HTTP server", args)
conf = CONFIG.copy()
if conf["http.server.class"]:
make_child = utils.import_class(conf["http.server.class"])
server = make_child(POLLER)
else:
server = HTTP_SERVER
server.configure(conf)
if conf["http.server.rootdir"] == ".":
conf["http.server.rootdir"] = os.path.abspath(".")
for port in conf["http.server.ports"].split(","):
if port:
server.listen((conf["http.server.address"], int(port)))
if conf["http.server.daemonize"]:
system.change_dir()
system.go_background()
system.write_pidfile()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
common.main("agent", "Run in background, periodically run tests", args)
conf = CONFIG.copy()
privacy.complain_if_needed()
if conf["agent.api"]:
server = HTTP_SERVER
LOG.debug("* API server root directory: %s" % WWW)
conf["http.server.rootdir"] = WWW
conf["http.server.ssi"] = True
conf["http.server.bind_or_die"] = True
server.configure(conf)
server.register_child(ServerAPI(POLLER), "/api")
server.listen((conf["agent.api.address"], conf["agent.api.port"]))
if conf["agent.daemonize"]:
system.change_dir()
system.go_background()
system.write_pidfile()
LOG.redirect()
if conf["agent.use_syslog"]:
LOG.redirect()
system.drop_privileges(LOG.error)
#
# When we run as an agent we also save logs into
# the database, to easily access and show them via
# the web user interface.
#
LOG.use_database()
if conf["agent.rendezvous"]:
client = ClientRendezvous(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main():
Sflag = False
options, arguments = getopt.getopt(sys.argv[1:], "S")
for key, value in options:
if key == "-S":
Sflag = True
# don't clobber my local database
DATABASE.set_path(":memory:")
DATABASE.connect()
if Sflag:
server = ServerSpeedtest(POLLER)
server.configure({"speedtest.negotiate.daemonize": False})
server.listen(("127.0.0.1", 8080))
else:
speedtest_again()
POLLER.loop()