def _do_run_queue(self, first_elem):
''' Actually run first element in queue '''
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
raise RuntimeError('runner_core: bad privacy settings')
elif first_elem[0] == 'rendezvous':
runner_rendezvous.run(conf['agent.master'], '9773')
elif first_elem[0] == 'speedtest':
uri = RUNNER_TESTS.test_to_negotiate_uri('speedtest')
conf['speedtest.client.uri'] = uri
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
elif first_elem[0] == 'bittorrent':
uri = RUNNER_TESTS.test_to_negotiate_uri('bittorrent')
conf['bittorrent._uri'] = uri
bittorrent.run(POLLER, conf)
elif first_elem[0] == 'dload':
RunnerDload(first_elem[2])
elif first_elem[0] == 'raw':
address = RUNNER_HOSTS.get_random_host()
handler = RawNegotiate()
handler.connect((address, 8080), CONFIG['prefer_ipv6'], 0, {})
elif first_elem[0] == 'mlab-ns':
handler = RunnerMlabns()
if not first_elem[2]:
extra = {'policy': ''} # get closest server by default
else:
extra = first_elem[2]
handler.connect(('mlab-ns.appspot.com', 80),
CONFIG['prefer_ipv6'], 0, extra)
elif first_elem[0] in self.dynamic_tests:
address = RUNNER_HOSTS.get_random_host()
port = 80 # XXX
self.dynamic_tests[first_elem[0]]["test_func"]({
"address": address,
"conf": CONFIG.copy(),
"poller": POLLER,
"port": port,
})
else:
raise RuntimeError('runner_core: asked to run an unknown test')
def main(args):
CONFIG.register_descriptions({
"speedtest.client.uri": "Base URI to connect to",
"speedtest.client.nconn": "Number of concurrent connections to use",
"speedtest.client.latency_tries": "Number of latency measurements",
})
common.main("speedtest.client", "Speedtest client", args)
conf = CONFIG.copy()
#
# If possible use the runner, which will execute the
# test in the context of the neubot daemon. Then exit
# to bypass the POLLER.loop() invokation that is below
# here.
# If the runner fails, fallback to the usual code path,
# which executes the test in the context of the local
# process.
# Set 'runned.enabled' to 0 to bypass the runner and
# run the test locally.
#
if (utils.intify(conf['runner.enabled']) and
runner_clnt.runner_client(conf["agent.api.address"],
conf["agent.api.port"],
LOG.noisy, "speedtest")):
sys.exit(0)
LOG.info('Will run the test in the local context...')
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
CONFIG.register_descriptions({
"speedtest.negotiate.address": "Address to listen to",
"speedtest.negotiate.auth_only": "Enable doing tests for authorized clients only",
"speedtest.negotiate.daemonize": "Enable going in background",
"speedtest.negotiate.port": "Port to listen to",
})
common.main("speedtest.negotiate", "Speedtest negotiation server", args)
conf = CONFIG.copy()
server = ServerSpeedtest(POLLER)
server.configure(conf)
server.listen((conf["speedtest.negotiate.address"],
conf["speedtest.negotiate.port"]))
if conf["speedtest.negotiate.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def __init__(self, ctx):
''' Download a file '''
self.ctx = ctx
ClientHTTP.__init__(self, POLLER)
self.configure(CONFIG.copy())
logging.debug('runner_dload: GET %s', self.ctx['uri'])
self.connect_uri(self.ctx['uri'])
def start_api(address=None, port=None):
''' Starts API for background module '''
logging.debug('background_api: starting API server...')
# Honor /etc/neubot/api
settings = utils_rc.parse_safe(utils_hier.APIFILEPATH)
if not address:
address = settings.get('address', '::1 127.0.0.1')
if not port:
port = settings.get('port', '9774')
# Configure HTTP server
conf = CONFIG.copy()
logging.debug('background_api: API server rootdir: %s',
utils_hier.WWWDIR)
conf['http.server.rootdir'] = utils_hier.WWWDIR
conf['http.server.ssi'] = True
conf['http.server.bind_or_die'] = True
HTTP_SERVER.configure(conf)
# Bind HTTP server to API server
HTTP_SERVER.register_child(API_SERVER, '/api')
# Bind HTTP server to address and port
HTTP_SERVER.listen((address, port))
logging.debug('background_api: starting API server... done')
def main(args):
CONFIG.register_descriptions({
"rendezvous.server.address": "Set rendezvous server address",
"rendezvous.server.daemonize": "Enable daemon behavior",
"rendezvous.server.ports": "List of rendezvous server ports",
"rendezvous.server.update_uri": "Where to download updates from",
"rendezvous.server.update_version": "Update Neubot version number",
"rendezvous.geoip_wrapper.country_database": "Path of the GeoIP country database",
"rendezvous.server.default": "Default test server to use",
})
common.main("rendezvous.server", "Rendezvous server", args)
conf = CONFIG.copy()
HTTP_SERVER.configure(conf)
for port in conf["rendezvous.server.ports"].split(","):
HTTP_SERVER.listen((conf["rendezvous.server.address"], int(port)))
# Really start this module
run(POLLER, conf)
if conf["rendezvous.server.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
''' main() of this module '''
CONFIG.register_descriptions({
"http.client.class": "Specify alternate ClientHTTP-like class",
"http.client.method": "Specify alternate HTTP method",
"http.client.stdout": "Enable writing response to stdout",
"http.client.uri": "Specify URI to download from/upload to",
})
common.main("http.client", "Simple Neubot HTTP client", args)
conf = CONFIG.copy()
make_client = TestClient
if conf["http.client.class"]:
make_client = utils.import_class(conf["http.client.class"])
if not conf["http.client.uri"]:
sys.stdout.write("Please, specify URI via -D http.client.uri=URI\n")
sys.exit(0)
client = make_client(POLLER)
client.configure(conf)
client.connect_uri(conf["http.client.uri"])
POLLER.loop()
sys.exit(0)
def main(args):
''' main() function of this module '''
CONFIG.register_descriptions({
"http.server.address": "Address to listen to",
"http.server.class": "Use alternate ServerHTTP-like class",
"http.server.mime": "Enable code that guess mime types",
"http.server.ports": "List of ports to listen to",
"http.server.rootdir": "Root directory for static pages",
"http.server.ssi": "Enable server-side includes",
})
common.main("http.server", "Neubot simple HTTP server", args)
conf = CONFIG.copy()
HTTP_SERVER.configure(conf)
if conf["http.server.rootdir"] == ".":
conf["http.server.rootdir"] = os.path.abspath(".")
for port in conf["http.server.ports"].split(","):
if port:
HTTP_SERVER.listen((conf["http.server.address"], int(port)))
POLLER.loop()
def __init__(self, ctx):
""" Download a file """
self.ctx = ctx
ClientHTTP.__init__(self, POLLER)
self.configure(CONFIG.copy())
logging.info("runner_dload: connecting to %s", self.ctx["uri"])
self.connect_uri(self.ctx["uri"])
def main(args):
""" Main function """
if not system.has_enough_privs():
sys.exit('FATAL: you must be root')
common.main("agent", "Run in background, periodically run tests", args)
conf = CONFIG.copy()
privacy.complain_if_needed()
BACKEND.use_backend("neubot")
BACKEND.datadir_init()
# FIXME We're ignoring agent.api.{address,port} that are now
# deprecated and should be removed soon.
background_api.start_api()
if conf["agent.daemonize"]:
LOG.redirect()
system.go_background()
if conf["agent.use_syslog"]:
LOG.redirect()
#
# When we run as an agent we also save logs into
# the database, to easily access and show them via
# the web user interface.
#
LOG.use_database()
logging.info('%s for POSIX: starting up', utils_version.PRODUCT)
system.drop_privileges()
if os.getuid() == 0 or os.geteuid() == 0:
logging.error('agent: still running as root')
os._exit(1)
if conf["agent.rendezvous"]:
BACKGROUND_RENDEZVOUS.start()
POLLER.loop()
logging.info('%s for POSIX: shutting down', utils_version.PRODUCT)
LOG.writeback()
#
# Make sure that we do not leave the database
# in an inconsistent state.
#
DATABASE.close()
def main(args):
CONFIG.register_descriptions({
"api.client.address": "Set address to connect to",
"api.client.port": "Set port to connect to",
})
common.main("api.client", "Minimal client for JSON API", args)
client = APIStateTracker(POLLER)
client.configure(CONFIG.copy())
client.loop()
def main(args):
CONFIG.register_descriptions({
# General variables
"net.stream.certfile": "Set SSL certfile path",
"net.stream.ipv6": "Enable IPv6",
"net.stream.key": "Set key for ARC4",
"net.stream.secure": "Enable SSL",
"net.stream.server_side": "Enable SSL server-side mode",
"net.stream.rcvbuf": "Set sock recv buffer (0 = use default)",
"net.stream.sndbuf": "Set sock send buffer (0 = use default)",
# For main()
"net.stream.address": "Set client or server address",
"net.stream.chunk": "Chunk written by each write",
"net.stream.clients": "Set number of client connections",
"net.stream.daemonize": "Enable daemon behavior",
"net.stream.duration": "Set duration of a test",
"net.stream.listen": "Enable server mode",
"net.stream.port": "Set client or server port",
"net.stream.proto": "Set proto (chargen, discard, or echo)",
})
common.main("net.stream", "TCP bulk transfer test", args)
conf = CONFIG.copy()
endpoint = (conf["net.stream.address"], conf["net.stream.port"])
if not conf["net.stream.proto"]:
if conf["net.stream.listen"]:
conf["net.stream.proto"] = "chargen"
else:
conf["net.stream.proto"] = "discard"
elif conf["net.stream.proto"] not in ("chargen", "discard", "echo"):
common.write_help(sys.stderr, "net.stream", "TCP bulk transfer test")
sys.exit(1)
handler = GenericHandler(POLLER)
handler.configure(conf)
if conf["net.stream.listen"]:
if conf["net.stream.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
conf["net.stream.server_side"] = True
handler.listen(endpoint)
else:
handler.connect(endpoint, count=conf["net.stream.clients"])
POLLER.loop()
sys.exit(0)
def main(args):
""" Main function """
try:
options, arguments = getopt.getopt(args[1:], "6A:fp:v")
except getopt.error:
sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]")
if arguments:
sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]")
prefer_ipv6 = 0
address = "master.neubot.org"
force = 0
port = 8080
noisy = 0
for name, value in options:
if name == "-6":
prefer_ipv6 = 1
elif name == "-A":
address = value
elif name == "-f":
force = 1
elif name == "-p":
port = int(value)
elif name == "-v":
noisy = 1
if os.path.isfile(DATABASE.path):
DATABASE.connect()
CONFIG.merge_database(DATABASE.connection())
else:
logging.warning("speedtest: database file is missing: %s", DATABASE.path)
BACKEND.use_backend("null")
if noisy:
log.set_verbose()
conf = CONFIG.copy()
conf["speedtest.client.uri"] = "http://%s:%d/" % (address, port)
conf["prefer_ipv6"] = prefer_ipv6
if not force:
if runner_clnt.runner_client(conf["agent.api.address"], conf["agent.api.port"], CONFIG["verbose"], "speedtest"):
sys.exit(0)
logging.warning("speedtest: failed to contact Neubot; is Neubot running?")
sys.exit(1)
logging.info("speedtest: run the test in the local process context...")
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
CONFIG.register_descriptions({
"speedtest.client.uri": "Base URI to connect to",
"speedtest.client.nconn": "Number of concurrent connections to use",
"speedtest.client.latency_tries": "Number of latency measurements",
})
common.main("speedtest.client", "Speedtest client", args)
conf = CONFIG.copy()
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def _do_run_queue(self):
''' Actually run first element in queue '''
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
raise RuntimeError('Bad privacy settings')
# Run rendezvous
elif self.queue[0][0] == 'rendezvous':
uri = "http://%s:9773/rendezvous" % conf['agent.master']
runner_rendezvous.run(uri)
# Run speedtest
elif self.queue[0][0] == 'speedtest':
uri = RUNNER_TESTS.test_to_negotiate_uri('speedtest')
#
# If we have no negotiate URI for this test, possibly
# because we are offline, abort it.
#
if not uri:
raise RuntimeError('No negotiate URI for speedtest')
conf['speedtest.client.uri'] = uri
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
# Run bittorrent
elif self.queue[0][0] == 'bittorrent':
uri = RUNNER_TESTS.test_to_negotiate_uri('bittorrent')
#
# If we have no negotiate URI for this test, possibly
# because we are offline, abort it.
#
if not uri:
raise RuntimeError('No negotiate URI for bittorrent')
conf['bittorrent._uri'] = uri
bittorrent.run(POLLER, conf)
# Run dload
elif self.queue[0][0] == 'dload':
RunnerDload(self.queue[0][2])
# Safety net
else:
raise RuntimeError('Asked to run an unknown test')
def main(args):
CONFIG.register_descriptions({
"rendezvous.client.debug": "Do not perform any test",
"rendezvous.client.version": "Set rendezvous client version",
})
common.main("rendezvous.client", "Rendezvous client", args)
conf = CONFIG.copy()
client = ClientRendezvous(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def check(updates):
''' Raises ConfigError if the user is trying to update the
privacy settings in a wrong way '''
# Merge a copy of config with privacy settings
conf = CONFIG.copy()
for key in PRIVACYKEYS:
if key in updates:
conf[key] = utils.intify(updates[key])
# Extract privacy settings from such copy
informed = utils.intify(conf.get("privacy.informed", 0))
can_collect = utils.intify(conf.get("privacy.can_collect", 0))
can_share = utils.intify(conf.get("privacy.can_share", 0))
if not informed:
# When you're not informed you cannot raise the other settings
if can_collect or can_share:
raise ConfigError("You cannot set can_collect or can_share "
"without asserting that you are informed")
else:
# It's not possible to share if you don't agree to collect
if can_share and not can_collect:
raise ConfigError("You cannot set can_share without also "
"setting can_collect (how are we supposed "
"to share what we cannot collect)?")
# You must give the can_collect bit
if not can_collect:
raise ConfigError("You must agree to collect or Neubot "
"cannot work. You should uninstall Neubot "
"if you don't want it to collect")
# You cannot remove the informed bit
if utils.intify(CONFIG['privacy.informed']) and not informed:
raise ConfigError("Once you're informed you cannot step back")
# You cannot remove the can_collect bit
if utils.intify(CONFIG['privacy.can_collect']) and not can_collect:
raise ConfigError("You can't remove the can_collect bit because "
"Neubot cannot work. You should uninstall Neubot "
"if you don't want it to collect")
def run_queue(self):
''' If possible run the first test in queue '''
# Adapted from neubot/rendezvous/client.py
if not self.queue:
return
if self.running:
return
#
# Subscribe BEFORE starting the test, otherwise we
# may miss the 'testdone' event if the connection
# to the negotiator service fails, and we will stay
# stuck forever.
#
NOTIFIER.subscribe('testdone', self.test_done)
# Prevent concurrent tests
self.running = True
# Make a copy of current settings
conf = CONFIG.copy()
# Make sure we abide to M-Lab policy
if privacy.count_valid(conf, 'privacy.') != 3:
privacy.complain()
NOTIFIER.publish('testdone')
# Run speedtest
elif self.queue[0][0] == 'speedtest':
conf['speedtest.client.uri'] = self.queue[0][1]
client = ClientSpeedtest(POLLER)
client.configure(conf)
client.connect_uri()
# Run bittorrent
elif self.queue[0][0] == 'bittorrent':
conf['bittorrent._uri'] = self.queue[0][1]
bittorrent.run(POLLER, conf)
# Safety net
else:
LOG.error('Asked to run an unknown test')
NOTIFIER.publish('testdone')
def main(args):
config.register_descriptions()
common.main("bittorrent", "Neubot BitTorrent module", args)
conf = CONFIG.copy()
config.finalize_conf(conf)
if conf["bittorrent.listen"]:
#
# If we need to negotiate and we're runing
# standalone we also need to bring up the
# global HTTP server.
#
if conf["bittorrent.negotiate"]:
HTTP_SERVER.configure(conf)
HTTP_SERVER.listen((conf["bittorrent.address"],
conf["bittorrent.negotiate.port"]))
conf["negotiate.listen"] = True
negotiate.run(POLLER, conf)
#
# Drop privileges after listen() so we can
# bind() to privileged ports
#
if conf["bittorrent.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
else:
#
# When we're connecting to a remote host to perform a test
# we want Neubot to quit at the end of the test. When this
# happens the test code publishes the "testdone" event, so
# here we prepare to intercept the event and break our main
# loop.
#
NOTIFIER.subscribe("testdone", lambda event, ctx: POLLER.break_loop())
run(POLLER, conf)
POLLER.loop()
def main(args):
common.main("speedtest.server", "Speedtest Test Server", args)
conf = CONFIG.copy()
server = ServerTest(POLLER)
server.configure(conf)
server.listen((conf["speedtest.server.address"],
conf["speedtest.server.port"]))
if conf["speedtest.server.daemonize"]:
system.change_dir()
system.go_background()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
CONFIG.register_descriptions({
# General variables
"net.stream.certfile": "Set SSL certfile path",
"net.stream.secure": "Enable SSL",
"net.stream.server_side": "Enable SSL server-side mode",
# For main()
"net.stream.address": "Set client or server address",
"net.stream.chunk": "Chunk written by each write",
"net.stream.clients": "Set number of client connections",
"net.stream.duration": "Set duration of a test",
"net.stream.listen": "Enable server mode",
"net.stream.port": "Set client or server port",
"net.stream.proto": "Set proto (chargen, discard, or echo)",
})
common.main("net.stream", "TCP bulk transfer test", args)
conf = CONFIG.copy()
endpoint = (conf["net.stream.address"], conf["net.stream.port"])
if not conf["net.stream.proto"]:
if conf["net.stream.listen"]:
conf["net.stream.proto"] = "chargen"
else:
conf["net.stream.proto"] = "discard"
elif conf["net.stream.proto"] not in ("chargen", "discard", "echo"):
common.write_help(sys.stderr, "net.stream", "TCP bulk transfer test")
sys.exit(1)
handler = GenericHandler(POLLER)
handler.configure(conf)
if conf["net.stream.listen"]:
conf["net.stream.server_side"] = True
handler.listen(endpoint)
else:
handler.connect(endpoint, count=conf["net.stream.clients"])
POLLER.loop()
sys.exit(0)
def start(address, port):
''' Starts API for background module '''
logging.debug('background_api: starting API server...')
# Configure HTTP server
conf = CONFIG.copy()
logging.debug('background_api: API server rootdir: %s', WWW)
conf['http.server.rootdir'] = WWW
conf['http.server.ssi'] = True
conf['http.server.bind_or_die'] = True
HTTP_SERVER.configure(conf)
# Bind HTTP server to API server
HTTP_SERVER.register_child(API_SERVER, '/api')
# Bind HTTP server to address and port
HTTP_SERVER.listen((address, port))
logging.debug('background_api: starting API server... done')
def main(args):
common.main("agent", "Run in background, periodically run tests", args)
conf = CONFIG.copy()
privacy.complain_if_needed()
if conf["agent.api"]:
server = HTTP_SERVER
LOG.debug("* API server root directory: %s" % WWW)
conf["http.server.rootdir"] = WWW
conf["http.server.ssi"] = True
conf["http.server.bind_or_die"] = True
server.configure(conf)
server.register_child(ServerAPI(POLLER), "/api")
server.listen((conf["agent.api.address"], conf["agent.api.port"]))
if conf["agent.daemonize"]:
system.change_dir()
system.go_background()
system.write_pidfile()
LOG.redirect()
if conf["agent.use_syslog"]:
LOG.redirect()
system.drop_privileges(LOG.error)
#
# When we run as an agent we also save logs into
# the database, to easily access and show them via
# the web user interface.
#
LOG.use_database()
if conf["agent.rendezvous"]:
client = ClientRendezvous(POLLER)
client.configure(conf)
client.connect_uri()
POLLER.loop()
def main(args):
''' main() function of this module '''
CONFIG.register_descriptions({
"http.server.address": "Address to listen to",
"http.server.class": "Use alternate ServerHTTP-like class",
"http.server.daemonize": "Run in background as a daemon",
"http.server.mime": "Enable code that guess mime types",
"http.server.ports": "List of ports to listen to",
"http.server.rootdir": "Root directory for static pages",
"http.server.ssi": "Enable server-side includes",
})
common.main("http.server", "Neubot simple HTTP server", args)
conf = CONFIG.copy()
if conf["http.server.class"]:
make_child = utils.import_class(conf["http.server.class"])
server = make_child(POLLER)
else:
server = HTTP_SERVER
server.configure(conf)
if conf["http.server.rootdir"] == ".":
conf["http.server.rootdir"] = os.path.abspath(".")
for port in conf["http.server.ports"].split(","):
if port:
server.listen((conf["http.server.address"], int(port)))
if conf["http.server.daemonize"]:
system.change_dir()
system.go_background()
system.write_pidfile()
LOG.redirect()
system.drop_privileges(LOG.error)
POLLER.loop()
def main(args):
''' Will initialize privacy settings '''
CONFIG.register_descriptions({
'privacy.init_informed': "You've read privacy policy",
'privacy.init_can_collect': 'We can collect your IP address',
'privacy.init_can_share': 'We can share your IP address',
'privacy.overwrite': 'Overwrite old settings',
})
common.main('privacy', 'Initialize privacy settings', args)
conf = CONFIG.copy()
if not conf['privacy.informed'] or conf['privacy.overwrite']:
dictonary = {
'privacy.informed': conf['privacy.init_informed'],
'privacy.can_collect': conf['privacy.init_can_collect'],
'privacy.can_share': conf['privacy.init_can_share'],
}
table_config.update(DATABASE.connection(), dictonary.iteritems())
DATABASE.connection().commit()
def main(args):
''' Main function '''
CONFIG.register_descriptions({
"rendezvous.server.address": "Set rendezvous server address",
"rendezvous.server.ports": "List of rendezvous server ports",
"rendezvous.server.update_version": "Update Neubot version number",
"rendezvous.geoip_wrapper.country_database": \
"Path of the GeoIP country database",
"rendezvous.server.default": "Default test server to use",
})
common.main("rendezvous.server", "Rendezvous server", args)
conf = CONFIG.copy()
HTTP_SERVER.configure(conf)
for port in conf["rendezvous.server.ports"].split(","):
HTTP_SERVER.listen((conf["rendezvous.server.address"], int(port)))
# Really start this module
run()
POLLER.loop()
def main(args):
''' Main function '''
try:
options, arguments = getopt.getopt(args[1:], 'lO:v')
except getopt.error:
sys.exit('usage: neubot bittorrent_peer [-lv] [-O setting]')
if arguments:
sys.exit('usage: neubot bittorrent_peer [-lv] [-O setting]')
settings = [ 'address "127.0.0.1 ::1"',
'port 6881',
'version 1' ]
listener = False
for name, value in options:
if name == '-l':
listener = True
elif name == '-O':
settings.append(value)
elif name == '-v':
CONFIG['verbose'] = 1
settings = utils_rc.parse_safe(iterable=settings)
config_copy = CONFIG.copy()
config.finalize_conf(config_copy)
peer = PeerNeubot(POLLER)
peer.configure(config_copy) # BLEAH
peer.version = int(settings['version'])
if not listener:
peer.connect((settings['address'], int(settings['port'])))
else:
peer.listen((settings['address'], int(settings['port'])))
POLLER.loop()
def main(args):
""" Main function """
try:
options, arguments = getopt.getopt(args[1:], "6A:b:d:flnp:v")
except getopt.error:
sys.exit(USAGE)
if arguments:
sys.exit(USAGE)
prefer_ipv6 = 0
address = "127.0.0.1"
backend = "volatile"
datadir = None # means: pick the default
force = 0
listen = 0
negotiate = 1
port = 80
noisy = 0
for name, value in options:
if name == "-6":
prefer_ipv6 = 1
elif name == "-A":
address = value
elif name == "-b":
backend = value
elif name == "-d":
datadir = value
elif name == "-f":
force = 1
elif name == "-l":
listen = 1
elif name == "-n":
negotiate = 0
elif name == "-p":
port = int(value)
elif name == "-v":
noisy = 1
if noisy:
log.set_verbose()
conf = CONFIG.copy()
BACKEND.use_backend(backend)
BACKEND.datadir_init(None, datadir)
if listen:
if not negotiate:
server = DASHServerSmpl(POLLER)
server.configure(conf)
server.listen((address, port))
else:
# Code adapted from neubot/server.py
conf["http.server.rootdir"] = ""
server = ServerHTTP(POLLER)
server.configure(conf)
server.listen((address, port))
negotiate_server = NegotiateServer(POLLER)
negotiate_server.configure(conf)
server.register_child(negotiate_server, "/negotiate")
server.register_child(negotiate_server, "/collect")
dash_negotiate_server = DASHNegotiateServer()
negotiate_server.register_module("dash", dash_negotiate_server)
dash_server = DASHServerGlue(POLLER, dash_negotiate_server)
dash_server.configure(conf)
server.register_child(dash_server, "/dash")
elif not force:
result = runner_clnt.runner_client(CONFIG["agent.api.address"],
CONFIG["agent.api.port"], CONFIG["verbose"], "dash")
if result:
sys.exit(0)
logging.warning("dash: failed to contact Neubot; is Neubot running?")
sys.exit(1)
else:
if negotiate:
client = DASHNegotiateClient(POLLER)
else:
client = DASHClientSmpl(POLLER, None, DASH_RATES)
client.configure(conf)
client.connect((address, port))
POLLER.loop()
def main(args):
""" Generate private key and certificate file for Neubot server """
CONFIG.register_descriptions({
"net.CA.bits": "Set private key bits number",
"net.CA.cacert": "Set certificate file path",
"net.CA.days": "Set days before expire",
"net.CA.privkey": "Set private key file path",
})
common.main("net.CA", "generate test certificates", args)
conf = CONFIG.copy()
#
# We need to be root because we play with file and
# directories permissions and ownership which, in the
# common case, cannot be done by other users.
#
if os.getuid():
sys.exit("This command must be invoked as root")
#
# Force a standard umask but note that we will
# override perms when needed.
# Create the base directory and allow root to
# populate and others just to read and list.
#
os.umask(0022)
if not os.path.exists(conf["net.CA.basedir"]):
os.mkdir(conf["net.CA.basedir"], 0755)
# Make paths absolute
conf["net.CA.cacert"] = os.sep.join([
conf["net.CA.basedir"],
conf["net.CA.cacert"]
])
conf["net.CA.privkey"] = os.sep.join([
conf["net.CA.basedir"],
conf["net.CA.privkey"]
])
# Generate RSA private key
genrsa = [ "openssl", "genrsa", "-out", conf["net.CA.privkey"],
str(conf["net.CA.bits"]) ]
LOG.debug("CA: exec: %s" % genrsa)
subprocess.call(genrsa)
# Generate self-signed certificate
req = [ "openssl", "req", "-new", "-x509", "-key", conf["net.CA.privkey"],
"-out", conf["net.CA.cacert"], "-days", str(conf["net.CA.days"]) ]
LOG.debug("CA: exec: %s" % req)
subprocess.call(req)
#
# Merge private key and self-signed certificate into
# the same file. While there, remove the original files
# from the filesystem.
#
certfile = os.sep.join([conf["net.CA.basedir"], "cert.pem"])
outfp = open(certfile, "w")
for key in ("net.CA.privkey", "net.CA.cacert"):
fpin = open(conf[key], "r")
os.unlink(conf[key])
outfp.write(fpin.read())
fpin.close()
outfp.close()
#
# Allow the `_neubot` user to read the file and set
# very restrictive permissions. Note that an attacker
# running as the `_neubot` user can steal or modify
# the on-disk private key quite easily. This is the
# basic flaw of the current SSL schema in Neubot.
#
rec = pwd.getpwnam("_neubot")
os.chown(certfile, rec.pw_uid, rec.pw_gid)
os.chmod(certfile, 0400)
def main(args):
""" Starts the server module """
if not system.has_enough_privs():
sys.exit('FATAL: you must be root')
try:
options, arguments = getopt.getopt(args[1:], 'A:b:D:dv')
except getopt.error:
sys.exit(USAGE)
if arguments:
sys.exit(USAGE)
address = ':: 0.0.0.0'
backend = 'mlab'
for name, value in options:
if name == '-A':
address = value
elif name == '-b':
backend = value
elif name == '-D':
name, value = value.split('=', 1)
if name not in VALID_MACROS:
sys.exit(USAGE)
if name != 'server.datadir': # XXX
value = int(value)
SETTINGS[name] = value
elif name == '-d':
SETTINGS['server.daemonize'] = 0
elif name == '-v':
CONFIG['verbose'] = 1
logging.debug('server: using backend: %s... in progress', backend)
if backend == 'mlab':
BACKEND.datadir_init(None, SETTINGS['server.datadir'])
BACKEND.use_backend('mlab')
elif backend == 'neubot':
DATABASE.connect()
BACKEND.use_backend('neubot')
elif backend == 'volatile':
BACKEND.use_backend('volatile')
else:
BACKEND.use_backend('null')
logging.debug('server: using backend: %s... complete', backend)
for name, value in SETTINGS.items():
CONFIG[name] = value
conf = CONFIG.copy()
#
# Configure our global HTTP server and make
# sure that we don't provide filesystem access
# even by mistake.
#
conf["http.server.rootdir"] = ""
HTTP_SERVER.configure(conf)
#
# New-new style: don't bother with abstraction and start the f*****g
# server by invoking its listen() method.
#
if CONFIG['server.raw']:
logging.debug('server: starting raw server... in progress')
RAW_SERVER_EX.listen((address, 12345),
CONFIG['prefer_ipv6'], 0, '')
logging.debug('server: starting raw server... complete')
if conf['server.skype']:
logging.debug('server: starting skype server... in progress')
SKYPE_SERVER_EX.listen((":: 0.0.0.0", 45678),
CONFIG['prefer_ipv6'], 0, '')
logging.debug('server: starting skype server... complete')
#
# New-style modules are started just setting a
# bunch of conf[] variables and then invoking
# their run() method in order to kick them off.
# This is now depricated in favor of the new-
# new style described above.
#
if conf["server.negotiate"]:
negotiate.run(POLLER, conf)
if conf["server.bittorrent"]:
conf["bittorrent.address"] = address
conf["bittorrent.listen"] = True
conf["bittorrent.negotiate"] = True
bittorrent.run(POLLER, conf)
if conf['server.speedtest']:
#conf['speedtest.listen'] = 1 # Not yet
#conf['speedtest.negotiate'] = 1 # Not yet
neubot.speedtest.wrapper.run(POLLER, conf)
# Migrating from old style to new style
if conf["server.rendezvous"]:
#conf["rendezvous.listen"] = True # Not yet
neubot.rendezvous.server.run()
#
# Historically Neubot runs on port 9773 and
# 8080 but we would like to switch to port 80
# in the long term period, because it's rare
# that they filter it.
# OTOH it looks like it's not possible to
# do that easily w/ M-Lab because the port
# is already taken.
#
ports = (80, 8080, 9773)
for port in ports:
HTTP_SERVER.listen((address, port))
#
# Start server-side API for Nagios plugin
# to query the state of the server.
# functionalities.
#
if conf["server.sapi"]:
server = ServerSideAPI(POLLER)
server.configure(conf)
HTTP_SERVER.register_child(server, "/sapi")
#
# Create localhost-only debug server
#
if CONFIG['server.debug']:
logging.info('server: Starting debug server at {127.0.0.1,::1}:9774')
server = DebugAPI(POLLER)
server.configure(conf)
server.listen(('127.0.0.1 ::1', 9774))
# Probe existing modules and ask them to attach to us
utils_modules.modprobe(None, "server", {
"http_server": HTTP_SERVER,
"negotiate_server": NEGOTIATE_SERVER,
})
#
# Go background and drop privileges,
# then enter into the main loop.
#
if conf["server.daemonize"]:
LOG.redirect()
system.go_background()
sigterm_handler = lambda signo, frame: POLLER.break_loop()
signal.signal(signal.SIGTERM, sigterm_handler)
logging.info('Neubot server -- starting up')
system.drop_privileges()
POLLER.loop()
logging.info('Neubot server -- shutting down')
utils_posix.remove_pidfile('/var/run/neubot.pid')
