def main(args): arguments = args[1:] if len(arguments) != 1: sys.stderr.write("Usage: tool_slice.py file\n") sys.exit(1) # Because below I'm lazy if not arguments[0].endswith(".sqlite3"): sys.stderr.write("error: Input file must end with .sqlite3 suffix\n") sys.exit(1) connection = sqlite3.connect(arguments[0]) connection.row_factory = sqlite3.Row since = TIMES.popleft() while TIMES: until = TIMES.popleft() output = sqlite3.connect(arguments[0].replace(".sqlite3", "_%d-%02d.sqlite3" % (since.year, since.month))) table_speedtest.create(output) cursor = connection.cursor() cursor.execute("""SELECT * FROM speedtest WHERE timestamp >= ? AND timestamp < ?;""", (since.strftime("%s"), until.strftime("%s"))) for row in cursor: table_speedtest.insert(output, dict(row), commit=False, override_timestamp=False) output.commit() since = until
def speedtest_store(self, message): ''' Saves the results of a speedtest test ''' DATABASE.connect() if DATABASE.readonly: logging.warning('backend_neubot: readonly database') return table_speedtest.insert(DATABASE.connection(), message)
def main(args): arguments = args[1:] if len(arguments) != 1: sys.stderr.write("Usage: tool_goodset.py file\n") sys.exit(1) # Because I'm lazy below if not arguments[0].endswith(".sqlite3"): sys.stderr.write("error: Input file must end with .sqlite3\n") sys.exit(1) outfile = arguments[0].replace(".sqlite3", "-goodset.sqlite3") sys.stderr.write("* Output database file: %s\n" % outfile) output = sqlite3.connect(outfile) table_speedtest.create(output) sys.stderr.write("* Processing file: %s... " % arguments[0]) input_dbm = sqlite3.connect(arguments[0]) input_dbm.row_factory = sqlite3.Row # Get the number of rows in the original database cursor = input_dbm.cursor() cursor.execute("SELECT COUNT(*) FROM speedtest;") total = cursor.next()[0] # Copy the goodset to the new database cursor = input_dbm.cursor() cursor.execute("""SELECT * FROM speedtest WHERE download_speed < ? AND latency < ?;""", (MAX_DOWNLOAD_SPEED, MAX_LATENCY)) for row in cursor: table_speedtest.insert(output, dict(row), commit=False, override_timestamp=False) sys.stderr.write("done\n") sys.stderr.write("* Committing changes to: %s\n" % outfile) output.commit() # Get the number of rows in the new database cursor = output.cursor() cursor.execute("SELECT COUNT(*) FROM speedtest;") goodset = cursor.next()[0] if total: sys.stdout.write("%d/%d (%.2f%%)\n" % (goodset, total, goodset * 100.0 / total)) else: sys.stdout.write("0/0 (0.00%)\n")
def runTest(self): """Make sure speedtest table works as expected""" connection = sqlite3.connect(":memory:") connection.row_factory = sqlite3.Row table_speedtest.create(connection) table_speedtest.create(connection) v = map(None, ResultIterator()) for d in v: table_speedtest.insert(connection, d, override_timestamp=False) v1 = table_speedtest.listify(connection) self.assertEquals(sorted(v), sorted(v1)) since = utils.timestamp() - 7 * 24 * 60 * 60 until = utils.timestamp() - 3 * 24 * 60 * 60 v2 = table_speedtest.listify(connection, since=since, until=until) self.assertTrue(len(v2) < len(v)) table_speedtest.prune(connection, until) self.assertTrue(len(table_speedtest.listify(connection)) < len(v1))
def collect_legacy(self, stream, request_body, request): ''' Invoked when we must save the result of a session ''' ident = str(hash(stream)) if ident not in self.clients: # # Before Neubot 0.4.2 we were using multiple connections # for speedtest, which were used both for testing and for # negotiating/collecting. Sometimes the connection used # to collect is not the one used to negotiate: the code # uses the one that terminates the upload first. # When this happens we inspect the Authorization header # before deciding the collect request is an abuse. # authorization = request['Authorization'] if authorization not in self.clients: raise RuntimeError('Not authorized to collect') else: LOG.warning('speedtest: working around multiple conns issue') ident = authorization # Note: no more than one collect per session self.clients.remove(ident) # # Backward compatibility: the variable name changed from # can_share to can_publish after Neubot 0.4.5 # if 'privacy_can_share' in request_body: request_body['privacy_can_publish'] = request_body[ 'privacy_can_share'] del request_body['privacy_can_share'] if privacy.collect_allowed(request_body): table_speedtest.insert(DATABASE.connection(), request_body) else: LOG.warning('* bad privacy settings: %s' % str(stream)) return {}
def speedtest_store(self, message): ''' Saves the results of a speedtest test ''' table_speedtest.insert(DATABASE.connection(), message)
def insertxxx(connection, obj, commit=True, override_timestamp=True): """ Hack to insert a result object into speedtest table, converting it into a dictionary. """ table_speedtest.insert(connection, obj_to_dict(obj), commit, override_timestamp)
def main(args): arguments = args[1:] if len(arguments) != 1: sys.stderr.write("Usage: tool_privacy.py file\n") sys.exit(1) # Because I'm lazy below if not arguments[0].endswith(".sqlite3"): sys.stderr.write("error: Input file name must end with .sqlite3\n") sys.exit(1) connection = sqlite3.connect(arguments[0]) connection.row_factory = sqlite3.Row # # Walk the database once and collect the most recent # permission for each unique identifier. We will then # use it to decide whether we can publish or not. # cursor = connection.cursor() cursor.execute("SELECT * FROM speedtest;") for row in cursor: PERMS[row['uuid']] = (row['privacy_informed'], row['privacy_can_collect'], row['privacy_can_share']) # # Build another database. Yes, from scratch. I don't # want leakage of your personal data to be possible, by # design. # output = sqlite3.connect(arguments[0].replace(".sqlite3", "-privacy.sqlite3")) table_speedtest.create(output) # # Walk again the original database and honour the # privacy permissions. We replace your Internet address # with all zeros, which is quite a good measure to # hide who you are. # total, can_share = 0, 0 cursor = connection.cursor() cursor.execute("SELECT * FROM speedtest;") for row in cursor: total = total + 1 dictionary = dict(row) # Honour permissions if PERMS[dictionary['uuid']] != (1, 1, 1): # # TODO Here it would be nice to geolocate the # client and resolv the provider to populate a # table that gives some information for the # clients that have not given permission. # dictionary['internal_address'] = "0.0.0.0" dictionary['real_address'] = "0.0.0.0" else: can_share = can_share + 1 # Override permissions (dictionary['privacy_informed'], dictionary['privacy_can_collect'], dictionary['privacy_can_share']) = PERMS[dictionary['uuid']] # NOTE commit=False or it will take an Age! table_speedtest.insert(output, dictionary, commit=False, override_timestamp=False) output.execute("VACUUM;") output.commit() # # Spit out per row statistics so we see how many rows we # can publish out of the total number of rows we have been # able to collect. # if total: sys.stdout.write("rows: %d/%d (%.02f%%)\n" % (can_share, total, (100.0 * can_share)/total)) else: sys.stdout.write("rows: 0/0 (0.0%)\n") # # Now tell the poor programmer what is the distribution # of privacy permissions one can find in the wild. # per_uuid= {} total_uuid = len(PERMS.values()) for tpl in PERMS.values(): if not tpl in per_uuid: per_uuid[tpl] = 0 per_uuid[tpl] += 1 if total_uuid: for perm in per_uuid: sys.stdout.write("perms: %s: %d/%d (%.02f%%)\n" % (perm, per_uuid[perm], total_uuid, (100.0 * per_uuid[perm])/total_uuid)) else: sys.stdout.write("perms: N/A: 0/0 (0.0%)\n") sys.exit(0)
def insert(self, row): dictionary = dict(row) table_speedtest.insert(self.dbm.connection(), dictionary, commit=False, override_timestamp=False) self.since = int(dictionary["timestamp"])