def speedtest_store(self, message): ''' Saves the results of a speedtest test ''' DATABASE.connect() if DATABASE.readonly: logging.warning('backend_neubot: readonly database') return table_speedtest.insert(DATABASE.connection(), message)
def main(args): ''' Main function ''' try: options, arguments = getopt.getopt(args[1:], 'f:nv') except getopt.error: sys.exit(USAGE) database_path = system.get_default_database_path() auto_discover = True for name, value in options: if name == '-f': database_path = value elif name == '-n': auto_discover = False elif name == '-v': CONFIG['verbose'] = 1 if len(arguments) != 1 and len(arguments) != 2: sys.exit(USAGE) DATABASE.set_path(database_path) CONFIG.merge_database(DATABASE.connection()) if len(arguments) == 2: RUNNER_TESTS.update({arguments[0]: [arguments[1]]}) ctx = {'uri': arguments[1]} else: ctx = None deferred = Deferred() deferred.add_callback(lambda param: None) RUNNER_CORE.run(arguments[0], deferred, auto_discover, ctx) POLLER.loop()
def main(args): ''' Main function ''' try: options, arguments = getopt.getopt(args[1:], 'f:n') except getopt.error: sys.exit('Usage: %s [-n] [-f database] test [negotiate_uri]' % args[0]) if len(arguments) != 1 and len(arguments) != 2: sys.exit('Usage: %s [-n] [-f database] test [negotiate_uri]' % args[0]) database_path = system.get_default_database_path() auto_rendezvous = True for name, value in options: if name == '-f': database_path = value elif name == '-n': auto_rendezvous = False DATABASE.set_path(database_path) CONFIG.merge_database(DATABASE.connection()) if len(arguments) == 2: RUNNER_TESTS.update({arguments[0]: [arguments[1]]}) ctx = {'uri': arguments[1]} else: ctx = None RUNNER_CORE.run(arguments[0], lambda *args: None, auto_rendezvous, ctx) POLLER.loop()
def main(args): """ Main function """ if not system.has_enough_privs(): sys.exit('FATAL: you must be root') common.main("agent", "Run in background, periodically run tests", args) conf = CONFIG.copy() privacy.complain_if_needed() BACKEND.use_backend("neubot") BACKEND.datadir_init() # FIXME We're ignoring agent.api.{address,port} that are now # deprecated and should be removed soon. background_api.start_api() if conf["agent.daemonize"]: LOG.redirect() system.go_background() if conf["agent.use_syslog"]: LOG.redirect() # # When we run as an agent we also save logs into # the database, to easily access and show them via # the web user interface. # LOG.use_database() logging.info('%s for POSIX: starting up', utils_version.PRODUCT) system.drop_privileges() if os.getuid() == 0 or os.geteuid() == 0: logging.error('agent: still running as root') os._exit(1) if conf["agent.rendezvous"]: BACKGROUND_RENDEZVOUS.start() POLLER.loop() logging.info('%s for POSIX: shutting down', utils_version.PRODUCT) LOG.writeback() # # Make sure that we do not leave the database # in an inconsistent state. # DATABASE.close()
def __create_empty(path): ''' This functions creates an empty Neubot database at @path, jusing Neubot internals to do that. ''' DATABASE.set_path(path) connection = DATABASE.connection() connection.commit() connection.close()
def main(args): """ Main function """ try: options, arguments = getopt.getopt(args[1:], "6A:np:vy") except getopt.error: sys.exit("usage: neubot skype [-6nvy] [-A address] [-p port]") if arguments: sys.exit("usage: neubot skype [-6nvy] [-A address] [-p port]") prefer_ipv6 = 0 # address = 'master.neubot.org' address = "localhost" runner = 1 port = 8080 noisy = 0 fakeprivacy = 0 for name, value in options: if name == "-6": prefer_ipv6 = 1 elif name == "-A": address = value elif name == "-n": runner = 0 elif name == "-p": port = int(value) elif name == "-v": noisy = 1 elif name == "-y": fakeprivacy = 1 if os.path.isfile(DATABASE.path): DATABASE.connect() CONFIG.merge_database(DATABASE.connection()) else: logging.warning("skype: database file is missing: %s", DATABASE.path) BACKEND.use_backend("null") if noisy: log.set_verbose() if runner: result = runner_clnt.runner_client( CONFIG["agent.api.address"], CONFIG["agent.api.port"], CONFIG["verbose"], "skype" ) if result: sys.exit(0) logging.info("skype: running the test in the local process context...") if not fakeprivacy and not privacy.allowed_to_run(): privacy.complain() logging.info("skype: otherwise use -y option to temporarily provide " "privacy permissions") sys.exit(1) handler = SkypeNegotiate() handler.connect((address, port), prefer_ipv6, 0, {}) POLLER.loop()
def main(args): """ Main function """ try: options, arguments = getopt.getopt(args[1:], "6A:fp:v") except getopt.error: sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]") if arguments: sys.exit("usage: neubot speedtest [-6fv] [-A address] [-p port]") prefer_ipv6 = 0 address = "master.neubot.org" force = 0 port = 8080 noisy = 0 for name, value in options: if name == "-6": prefer_ipv6 = 1 elif name == "-A": address = value elif name == "-f": force = 1 elif name == "-p": port = int(value) elif name == "-v": noisy = 1 if os.path.isfile(DATABASE.path): DATABASE.connect() CONFIG.merge_database(DATABASE.connection()) else: logging.warning("speedtest: database file is missing: %s", DATABASE.path) BACKEND.use_backend("null") if noisy: log.set_verbose() conf = CONFIG.copy() conf["speedtest.client.uri"] = "http://%s:%d/" % (address, port) conf["prefer_ipv6"] = prefer_ipv6 if not force: if runner_clnt.runner_client(conf["agent.api.address"], conf["agent.api.port"], CONFIG["verbose"], "speedtest"): sys.exit(0) logging.warning("speedtest: failed to contact Neubot; is Neubot running?") sys.exit(1) logging.info("speedtest: run the test in the local process context...") client = ClientSpeedtest(POLLER) client.configure(conf) client.connect_uri() POLLER.loop()
def main(name, descr, args): Eflag = False lflag = False try: options, arguments = getopt.getopt(args[1:], "D:Ef:lVv", ["help"]) except getopt.GetoptError: write_help(sys.stderr, name, descr) sys.exit(1) if arguments: write_help(sys.stderr, name, descr) sys.exit(1) verbose = 0 for key, value in options: if key == "-D": # No shortcuts because it grows too confusing CONFIG.register_property(value) elif key == "-E": Eflag = True elif key == "-f": DATABASE.set_path(value) elif key == "--help": write_help(sys.stdout, name, descr) sys.exit(0) elif key == "-l": lflag = True elif key == "-V": sys.stdout.write(VERSION + "\n") sys.exit(0) elif key == "-v": verbose = 1 DATABASE.connect() CONFIG.merge_database(DATABASE.connection()) if not Eflag: CONFIG.merge_environ() CONFIG.merge_properties() # Apply the setting after we've read database and environment if verbose: CONFIG['verbose'] = 1 if lflag: CONFIG.print_descriptions(sys.stdout) sys.exit(0)
def main(args): ''' Main() function ''' try: options, arguments = getopt.getopt(args[1:], '') except getopt.error: sys.exit('usage: neubot background_win32') if options or arguments: sys.exit('usage: neubot background_win32') # Read settings from database CONFIG.merge_database(DATABASE.connection()) # # Save logs into the database, to easily access # and show them via the web user interface. # LOG.use_database() # Complain if privacy settings are not OK privacy.complain_if_needed() background_api.start('127.0.0.1 ::1', '9774') BACKGROUND_RENDEZVOUS.start() __start_updater() POLLER.loop()
def got_response_collecting(self, stream, request, response): logging.info("BitTorrent: collecting ... done") if self.success: # # Always measure at the receiver because there is more # information at the receiver and also to make my friend # Enrico happier :-P. # The following is not a bug: it's just that the server # returns a result using the point of view of the client, # i.e. upload_speed is _our_ upload speed. # m = json.loads(response.body.read()) self.my_side["upload_speed"] = m["upload_speed"] upload = utils.speed_formatter(m["upload_speed"]) STATE.update("test_progress", "100%", publish=False) STATE.update("test_upload", upload) logging.info('BitTorrent: upload speed: %s', upload) if privacy.collect_allowed(self.my_side): if DATABASE.readonly: logging.warning('bittorrent_client: readonly database') else: table_bittorrent.insert(DATABASE.connection(), self.my_side) # Update the upstream channel estimate target_bytes = int(m["target_bytes"]) if target_bytes > 0: estimate.UPLOAD = target_bytes self.final_state = True stream.close()
def collect(self, m): btid = _make_btid(m["ident"]) if btid not in AUTH_PEERS: raise NegotiatorEOF() d = m["request_body"] result = AUTH_PEERS[btid] # # Note that the following is not a bug: it's just that # the server saves results using the point of view of the # client, i.e. upload_speed _is_ client's upload speed. # d["timestamp"] = result["timestamp"] d["upload_speed"] = result["upload_speed"] if privacy.collect_allowed(d): table_bittorrent.insert(DATABASE.connection(), d) # # After we've saved the result into the dictionary we # can add extra information we would like to return to # the client. # d["target_bytes"] = result["target_bytes"] m["response_body"] = d
def got_response_collecting(self, stream, request, response): LOG.complete() if self.success: # # Always measure at the receiver because there is more # information at the receiver and also to make my friend # Enrico happier :-P. # The following is not a bug: it's just that the server # returns a result using the point of view of the client, # i.e. upload_speed is _our_ upload speed. # m = json.loads(response.body.read()) self.my_side["upload_speed"] = m["upload_speed"] upload = utils.speed_formatter(m["upload_speed"]) STATE.update("test_upload", upload) if privacy.collect_allowed(self.my_side): table_bittorrent.insert(DATABASE.connection(), self.my_side) # Update the upstream channel estimate target_bytes = int(m["target_bytes"]) if target_bytes > 0: estimate.UPLOAD = target_bytes stream.close()
def api_data(stream, request, query): ''' Get data stored on the local database ''' since, until = -1, -1 test = '' dictionary = cgi.parse_qs(query) if "test" in dictionary: test = str(dictionary["test"][0]) if "since" in dictionary: since = int(dictionary["since"][0]) if "until" in dictionary: until = int(dictionary["until"][0]) if test == 'bittorrent': table = table_bittorrent elif test == 'speedtest': table = table_speedtest elif test == 'raw': table = table_raw else: raise NotImplementedTest("Test not implemented") indent, mimetype, sort_keys = None, "application/json", False if "debug" in dictionary and utils.intify(dictionary["debug"][0]): indent, mimetype, sort_keys = 4, "text/plain", True response = Message() lst = table.listify(DATABASE.connection(), since, until) body = json.dumps(lst, indent=indent, sort_keys=sort_keys) response.compose(code="200", reason="Ok", body=body, mimetype=mimetype) stream.send_response(request, response)
def listify(self): if self._use_database: lst = table_log.listify(DATABASE.connection()) lst.extend(self._queue) return lst else: return []
def _api_config(self, stream, request, query): response = Message() indent, mimetype, sort_keys = None, "application/json", False dictionary = cgi.parse_qs(query) if "debug" in dictionary and utils.intify(dictionary["debug"][0]): indent, mimetype, sort_keys = 4, "text/plain", True if request.method == "POST": s = request.body.read() updates = qs_to_dictionary(s) privacy.check(updates) # Very low barrier to prevent damage from kiddies if "agent.interval" in updates: interval = int(updates["agent.interval"]) if interval < 1380 and interval != 0: raise ConfigError("Bad agent.interval") CONFIG.merge_api(updates, DATABASE.connection()) STATE.update("config", updates) # Empty JSON b/c '204 No Content' is treated as an error s = "{}" else: s = json.dumps(CONFIG.conf, sort_keys=sort_keys, indent=indent) stringio = StringIO.StringIO(s) response.compose(code="200", reason="Ok", body=stringio, mimetype=mimetype) stream.send_response(request, response)
def api_results(stream, request, query): ''' Provide results for queried tests ''' since, until = -1, -1 test = '' dictionary = cgi.parse_qs(query) if dictionary.has_key("test"): test = str(dictionary["test"][0]) if dictionary.has_key("since"): since = int(dictionary["since"][0]) if dictionary.has_key("until"): until = int(dictionary["until"][0]) if test == 'bittorrent': table = table_bittorrent elif test == 'speedtest': table = table_speedtest else: raise NotImplementedTest("Test '%s' is not implemented" % test) indent, mimetype, sort_keys = None, "application/json", False if "debug" in dictionary and utils.intify(dictionary["debug"][0]): indent, mimetype, sort_keys = 4, "text/plain", True response = Message() lst = table.listify(DATABASE.connection(), since, until) body = json.dumps(lst, indent=indent, sort_keys=sort_keys) response.compose(code="200", reason="Ok", body=body, mimetype=mimetype) stream.send_response(request, response)
def main(args): ''' Main function ''' try: options, arguments = getopt.getopt(args[1:], '6A:fp:v') except getopt.error: sys.exit('usage: neubot raw [-6fv] [-A address] [-p port]') if arguments: sys.exit('usage: neubot raw [-6fv] [-A address] [-p port]') prefer_ipv6 = 0 address = 'master.neubot.org' force = 0 port = 8080 noisy = 0 for name, value in options: if name == '-6': prefer_ipv6 = 1 elif name == '-A': address = value elif name == '-f': force = 1 elif name == '-p': port = int(value) elif name == '-v': noisy = 1 if os.path.isfile(DATABASE.path): DATABASE.connect() CONFIG.merge_database(DATABASE.connection()) else: logging.warning('raw: database file is missing: %s', DATABASE.path) BACKEND.use_backend('null') if noisy: log.set_verbose() if not force: result = runner_clnt.runner_client(CONFIG['agent.api.address'], CONFIG['agent.api.port'], CONFIG['verbose'], 'raw') if result: sys.exit(0) logging.warning('raw: failed to contact Neubot; is Neubot running?') sys.exit(1) logging.info('raw: run the test in the local process context...') handler = RawNegotiate() handler.connect((address, port), prefer_ipv6, 0, {}) POLLER.loop()
def __main(args): ''' Initialize privacy settings ''' try: options, arguments = getopt.getopt(args[1:], 'D:f:Pt') except getopt.error: sys.exit(USAGE) if arguments: sys.exit(USAGE) settings = {} database_path = system.get_default_database_path() pflag = False testmode = False for name, value in options: if name == '-D': name, value = value.split('=', 1) if not name.startswith("privacy."): name = "privacy." + name settings[name] = value elif name == '-f': database_path = value elif name == '-P': pflag = True elif name == '-t': testmode = True if pflag: sys.exit(print_policy()) DATABASE.set_path(database_path) connection = DATABASE.connection() if testmode: sys.exit(test_settings(connection)) if settings: if DATABASE.readonly: sys.exit('ERROR: readonly database') sys.exit(update_settings(connection, settings)) sys.exit(print_settings(connection, database_path))
def main(args): ''' Main() function ''' try: options, arguments = getopt.getopt(args[1:], '') except getopt.error: sys.exit('usage: neubot background_win32') if options or arguments: sys.exit('usage: neubot background_win32') # Read settings from database CONFIG.merge_database(DATABASE.connection()) BACKEND.use_backend("neubot") BACKEND.datadir_init() # # Save logs into the database, to easily access # and show them via the web user interface. # LOG.use_database() logging.info('%s for Windows: starting up', utils_version.PRODUCT) # Complain if privacy settings are not OK privacy.complain_if_needed() background_api.start_api() BACKGROUND_RENDEZVOUS.start() __start_updater() POLLER.loop() logging.info('%s for Windows: shutting down', utils_version.PRODUCT) LOG.writeback() # # Make sure that we do not leave the database # in an inconsistent state. # DATABASE.close()
def main(args): ''' Main function ''' try: options, arguments = getopt.getopt(args[1:], 'f') except getopt.error: sys.exit('Usage: %s [-f database] test negotiate_uri' % args[0]) if len(arguments) != 2: sys.exit('Usage: %s [-f database] test negotiate_uri' % args[0]) database_path = system.get_default_database_path() for name, value in options: if name == '-f': database_path = value DATABASE.set_path(database_path) CONFIG.merge_database(DATABASE.connection()) run(arguments[0], arguments[1], lambda: None) POLLER.loop()
def __writeback(self): """Really commit pending log records into the database""" connection = DATABASE.connection() table_log.prune(connection, DAYS_AGO, commit=False) for record in self._queue: table_log.insert(connection, record, False) connection.commit() connection.execute("VACUUM;") connection.commit()
def do_collect(self, stream, request): self._speedtest_complete(request) s = request.body.read() m = marshal.unmarshal_object(s, "text/xml", compat.SpeedtestCollect) if privacy.collect_allowed(m): table_speedtest.insertxxx(DATABASE.connection(), m) response = Message() response.compose(code="200", reason="Ok") stream.send_response(request, response)
def main(): Sflag = False options, arguments = getopt.getopt(sys.argv[1:], "S") for key, value in options: if key == "-S": Sflag = True # don't clobber my local database DATABASE.set_path(":memory:") DATABASE.connect() if Sflag: server = ServerSpeedtest(POLLER) server.configure({"speedtest.negotiate.daemonize": False}) server.listen(("127.0.0.1", 8080)) else: speedtest_again() POLLER.loop()
def main(args): ''' Will initialize privacy settings ''' CONFIG.register_descriptions({ 'privacy.init_informed': "You've read privacy policy", 'privacy.init_can_collect': 'We can collect your IP address', 'privacy.init_can_share': 'We can share your IP address', 'privacy.overwrite': 'Overwrite old settings', }) common.main('privacy', 'Initialize privacy settings', args) conf = CONFIG.copy() if not conf['privacy.informed'] or conf['privacy.overwrite']: dictonary = { 'privacy.informed': conf['privacy.init_informed'], 'privacy.can_collect': conf['privacy.init_can_collect'], 'privacy.can_share': conf['privacy.init_can_share'], } table_config.update(DATABASE.connection(), dictonary.iteritems()) DATABASE.connection().commit()
def _writeback(self): """Really commit pending log records into the database""" connection = DATABASE.connection() table_log.prune(connection, DAYS_AGO, commit=False) for record in self._queue: table_log.insert(connection, record, False) connection.commit() now = utils.ticks() if now - self.last_vacuum > INTERVAL_VACUUM: connection.execute("VACUUM;") self.last_vacuum = now connection.commit()
def connection_ready(self, stream): m1 = SpeedtestCollect() m1.client = self.conf.get("uuid", "") m1.timestamp = utils.timestamp() m1.internalAddress = stream.myname[0] m1.realAddress = self.conf.get("speedtest.client.public_address", "") m1.remoteAddress = stream.peername[0] m1.latency = self.conf.get("speedtest.client.latency", 0.0) m1.downloadSpeed = self.conf.get("speedtest.client.download", 0.0) m1.uploadSpeed = self.conf.get("speedtest.client.upload", 0.0) m1.privacy_informed = self.conf.get("privacy.informed", 0) m1.privacy_can_collect = self.conf.get("privacy.can_collect", 0) m1.privacy_can_share = self.conf.get("privacy.can_publish", 0) # XXX m1.neubot_version = utils_version.NUMERIC_VERSION m1.platform = sys.platform m1.connectTime = sum(self.rtts) / len(self.rtts) # Test version (added Neubot 0.4.12) m1.testVersion = CONFIG["speedtest_test_version"] s = marshal.marshal_object(m1, "text/xml") stringio = StringIO.StringIO(s) # # Pass a dictionary because the function does not accept # anymore an object # if privacy.collect_allowed(m1.__dict__): if DATABASE.readonly: logging.warning("speedtest: readonly database") else: insertxxx(DATABASE.connection(), m1) request = Message() request.compose( method="POST", pathquery="/speedtest/collect", body=stringio, mimetype="application/xml", host=self.host_header, ) request["authorization"] = self.conf.get("speedtest.client.authorization", "") stream.send_request(request)
def connection_ready(self, stream): m1 = compat.SpeedtestCollect() m1.client = self.conf.get("uuid", "") m1.timestamp = utils.timestamp() m1.internalAddress = stream.myname[0] m1.realAddress = self.conf.get("speedtest.client.public_address", "") m1.remoteAddress = stream.peername[0] m1.latency = self.conf.get("speedtest.client.latency", 0.0) m1.downloadSpeed = self.conf.get("speedtest.client.download", 0.0) m1.uploadSpeed = self.conf.get("speedtest.client.upload", 0.0) m1.privacy_informed = self.conf.get("privacy.informed", 0) m1.privacy_can_collect = self.conf.get("privacy.can_collect", 0) m1.privacy_can_share = self.conf.get("privacy.can_share", 0) m1.neubot_version = LibVersion.to_numeric("0.4.2") m1.platform = sys.platform if self.measurer: m1.connectTime = self.measurer.measure_rtt()[0] # import pprint # if hasattr(self.measurer, "recv_hist"): # download = self.measurer.recv_hist.get("download", []) # pprint.pprint(download) # if hasattr(self.measurer, "send_hist"): # upload = self.measurer.send_hist.get("upload", []) # pprint.pprint(upload) s = marshal.marshal_object(m1, "text/xml") stringio = StringIO.StringIO(s) if privacy.collect_allowed(m1): table_speedtest.insertxxx(DATABASE.connection(), m1) request = Message() request.compose(method="POST", pathquery="/speedtest/collect", body=stringio, mimetype="application/xml", host=self.host_header) request["authorization"] = self.conf.get( "speedtest.client.authorization", "") stream.send_request(request)
def _api_speedtest(self, stream, request, query): since, until = -1, -1 dictionary = cgi.parse_qs(query) if dictionary.has_key("since"): since = int(dictionary["since"][0]) if dictionary.has_key("until"): until = int(dictionary["until"][0]) indent, mimetype, sort_keys = None, "application/json", False if "debug" in dictionary and utils.intify(dictionary["debug"][0]): indent, mimetype, sort_keys = 4, "text/plain", True response = Message() lst = table_speedtest.listify(DATABASE.connection(), since, until) s = json.dumps(lst, indent=indent, sort_keys=sort_keys) stringio = StringIO.StringIO(s) response.compose(code="200", reason="Ok", body=stringio, mimetype=mimetype) stream.send_response(request, response)
def collect(self, stream, request_body): ''' Invoked when we must save the result of a session ''' sha1 = self._stream_to_sha1(stream) if sha1 not in self.peers: raise RuntimeError('Not authorized to collect') else: # Note: no more than one collect per session result = self.peers[sha1] del self.peers[sha1] # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # if 'privacy_can_share' in request_body: request_body['privacy_can_publish'] = request_body[ 'privacy_can_share'] del request_body['privacy_can_share'] # # Note that the following is not a bug: it's just that # the server saves results using the point of view of the # client, i.e. upload_speed _is_ client's upload speed. # request_body['timestamp'] = result['timestamp'] request_body['upload_speed'] = result['upload_speed'] if privacy.collect_allowed(request_body): table_bittorrent.insert(DATABASE.connection(), request_body) else: logging.warning('* bad privacy settings: %s', str(stream)) # # After we've saved the result into the dictionary we # can add extra information we would like to return to # the client. # request_body['target_bytes'] = result['target_bytes'] return request_body
def collect_legacy(self, stream, request_body, request): ''' Invoked when we must save the result of a session ''' ident = str(hash(stream)) if ident not in self.clients: # # Before Neubot 0.4.2 we were using multiple connections # for speedtest, which were used both for testing and for # negotiating/collecting. Sometimes the connection used # to collect is not the one used to negotiate: the code # uses the one that terminates the upload first. # When this happens we inspect the Authorization header # before deciding the collect request is an abuse. # authorization = request['Authorization'] if authorization not in self.clients: raise RuntimeError('Not authorized to collect') else: LOG.warning('speedtest: working around multiple conns issue') ident = authorization # Note: no more than one collect per session self.clients.remove(ident) # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # if 'privacy_can_share' in request_body: request_body['privacy_can_publish'] = request_body[ 'privacy_can_share'] del request_body['privacy_can_share'] if privacy.collect_allowed(request_body): table_speedtest.insert(DATABASE.connection(), request_body) else: LOG.warning('* bad privacy settings: %s' % str(stream)) return {}