def cleanup(conn): """ To be called on exit """ # TODO: Stop other threads and close circuits plog("INFO", "Cleaning up...") conn.set_option("__LeaveStreamsUnattached", "0") conn.set_option("__DisablePredictedCircuits", "0") conn.close()
def choose_url(percentile): # TODO: Maybe we don't want to read the file *every* time? # Maybe once per slice? # Read in the bw auths file # here is a fine place to make sure we have bwfiles try: f = file("./data/bwfiles", "r") except IOError: write_file_list('./data') lines = [] valid = False for l in f.readlines(): if l == ".\n": valid = True break pair = l.split() lines.append((int(pair[0]), pair[1])) if not valid: plog("ERROR", "File size list is invalid!") for (pct, fname) in lines: if percentile < pct: return random.choice(urls) + fname raise PathSupport.NoNodesRemain("No nodes left for url choice!")
def setup_handler(out_dir, cookie_file): plog("INFO", "Connecting to Tor at " + TorUtil.control_host + ":" + str(TorUtil.control_port)) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TorUtil.control_host, TorUtil.control_port)) c = PathSupport.Connection(s) # c.debug(file(out_dir+"/control.log", "w", buffering=0)) c.authenticate_cookie(file(cookie_file, "r")) h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) # ignore existing streams ignore_streams(c, h) c.set_event_handler(h) # c.set_periodic_timer(2.0, "PULSE") c.set_events( [ TorCtl.EVENT_TYPE.STREAM, TorCtl.EVENT_TYPE.BW, TorCtl.EVENT_TYPE.NEWCONSENSUS, TorCtl.EVENT_TYPE.NEWDESC, TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW, ], True, ) atexit.register(cleanup) return (c, h)
def send_command_and_check(self, line): self.sock.send(line + '\r\n') reply = self.readline() if reply[:3] != '250': plog('ERROR', reply) raise MetatrollerException(reply) return reply
def setup_handler(out_dir, cookie_file): plog( 'INFO', 'Connecting to Tor at ' + TorUtil.control_host + ":" + str(TorUtil.control_port)) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TorUtil.control_host, TorUtil.control_port)) c = PathSupport.Connection(s) #c.debug(file(out_dir+"/control.log", "w", buffering=0)) c.authenticate_cookie(file(cookie_file, "r")) h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) # ignore existing streams ignore_streams(c, h) c.set_event_handler(h) #c.set_periodic_timer(2.0, "PULSE") c.set_events([ TorCtl.EVENT_TYPE.STREAM, TorCtl.EVENT_TYPE.BW, TorCtl.EVENT_TYPE.NEWCONSENSUS, TorCtl.EVENT_TYPE.NEWDESC, TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW ], True) atexit.register(cleanup) return (c, h)
def http_request(address): """ perform an http GET-request and return 1 for success or 0 for failure """ request = urllib2.Request(address) request.add_header("User-Agent", user_agent) try: reply = urllib2.urlopen(request) decl_length = reply.info().get("Content-Length") read_len = len(reply.read()) plog("DEBUG", "Read: " + str(read_len) + " of declared " + str(decl_length)) return 1 except (ValueError, urllib2.URLError): plog("ERROR", "The http-request address " + address + " is malformed") return 0 except (IndexError, TypeError): plog("ERROR", "An error occured while negotiating socks5 with Tor") return 0 except KeyboardInterrupt: raise KeyboardInterrupt except socks.Socks5Error, e: if e.value[0] == 6: plog("NOTICE", "Tor timed out our SOCKS stream request.") else: plog("ERROR", "An unknown HTTP error occured") traceback.print_exc() return 0
def heartbeat_event(self, event): if len(self.live_circs) < MAX_CIRCUITS: try: circ_id = self.c.extend_circuit() plog("INFO", "Launched circuit: "+str(circ_id)) except TorCtl.ErrorReply, e: plog("WARN", "Can't extend circuit: "+str(e))
def main(argv): TorUtil.read_config(argv[1]) ( start_pct, stop_pct, nodes_per_slice, save_every, circs_per_node, out_dir, max_fetch_time, tor_dir, sleep_start, sleep_stop, min_streams, pid_file_name, ) = read_config(argv[1]) if pid_file_name: pidfd = file(pid_file_name, "w") pidfd.write("%d\n" % os.getpid()) pidfd.close() try: (c, hdlr) = setup_handler(out_dir, tor_dir + "/control_auth_cookie") except Exception, e: traceback.print_exc() plog("WARN", "Can't connect to Tor: " + str(e))
def get_guards(c, n): # Get list of live routers sorted_rlist = filter(lambda r: not r.down, c.read_routers(c.get_network_status())) sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw)) list_len = len(sorted_rlist) for i in xrange(list_len): sorted_rlist[i].list_rank = i guard_rst = PathSupport.FlagsRestriction(["Guard"], []) if pct_start == 100: pct_rst = PathSupport.PercentileRestriction(0, pct_start, sorted_rlist) else: pct_rst = PathSupport.PercentileRestriction(pct_start, pct_start+PCT_SKIP, sorted_rlist) # XXX: Hrmm. UniformGenerator was broken? guard_gen = PathSupport.ExactUniformGenerator(sorted_rlist, PathSupport.NodeRestrictionList([guard_rst, pct_rst])) guard_gen.rewind() ggen = guard_gen.generate() # Generate 3 guards guards = [] for i in xrange(n): g = ggen.next() plog("NOTICE", str(pct_start)+"%: Generated guard $"+g.idhex+" with rank "+str(g.list_rank)+"/"+str(list_len)+" "+str(round((100.0*g.list_rank)/list_len, 1))+"% with flags "+str(g.flags)) guards.append(g) return guards
def http_request(address): ''' perform an http GET-request and return 1 for success or 0 for failure ''' request = urllib2.Request(address) request.add_header('User-Agent', user_agent) try: reply = urllib2.urlopen(request) decl_length = reply.info().get("Content-Length") read_len = len(reply.read()) plog("DEBUG", "Read: " + str(read_len) + " of declared " + str(decl_length)) return 1 except (ValueError, urllib2.URLError): plog('ERROR', 'The http-request address ' + address + ' is malformed') return 0 except (IndexError, TypeError): plog('ERROR', 'An error occured while negotiating socks5 with Tor') return 0 except KeyboardInterrupt: raise KeyboardInterrupt except socks.Socks5Error, e: if e.value[0] == 6: plog("NOTICE", "Tor timed out our SOCKS stream request.") else: plog('ERROR', 'An unknown HTTP error occured') traceback.print_exc() return 0
def __init__(self, line): # node_id=$DB8C6D8E0D51A42BDDA81A9B8A735B41B2CF95D1 bw=231000 diff=209281 nick=rainbowwarrior measured_at=1319822504 self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1) self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1) self.bw = int(re.search("[\s]+bw=([\S]+)[\s]*", line).group(1)) self.measured_at = int( re.search("[\s]*measured_at=([\S]+)[\s]*", line).group(1)) try: self.pid_error = float( re.search("[\s]*pid_error=([\S]+)[\s]*", line).group(1)) self.pid_error_sum = float( re.search("[\s]*pid_error_sum=([\S]+)[\s]*", line).group(1)) self.pid_delta = float( re.search("[\s]*pid_delta=([\S]+)[\s]*", line).group(1)) self.pid_bw = float( re.search("[\s]*pid_bw=([\S]+)[\s]*", line).group(1)) except: plog("NOTICE", "No previous PID data.") self.pid_bw = self.bw self.pid_error = 0 self.pid_delta = 0 self.pid_error_sum = 0 try: self.updated_at = int( re.search("[\s]*updated_at=([\S]+)[\s]*", line).group(1)) except: plog("INFO", "No updated_at field for " + self.nick + "=" + self.idhex) self.updated_at = self.measured_at
def print_circuits(self, list=None): """ Print out the circuits + some info, optionally pass a (sorted) list """ if list: circs = list else: circs = self.circuits.values() plog("INFO", "We have " + str(len(circs)) + " circuits:") for c in circs: print("+ " + c.to_string())
def heartbeat_event(self, event): if len(self.live_circs) < MAX_CIRCUITS: try: circ_id = self.c.extend_circuit() plog("INFO", "Launched circuit: " + str(circ_id)) except TorCtl.ErrorReply, e: plog("WARN", "Can't extend circuit: " + str(e))
def setup_handler(out_dir, cookie_file): plog("INFO", "Connecting to Tor at " + TorUtil.control_host + ":" + str(TorUtil.control_port)) s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TorUtil.control_host, TorUtil.control_port)) c = PathSupport.Connection(s) # c.debug(file(out_dir+"/control.log", "w", buffering=0)) c.authenticate_cookie(file(cookie_file, "r")) # f = c.get_option("__LeaveStreamsUnattached")[0] h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector) c.set_event_handler(h) # c.set_periodic_timer(2.0, "PULSE") c.set_events( [ TorCtl.EVENT_TYPE.STREAM, TorCtl.EVENT_TYPE.BW, TorCtl.EVENT_TYPE.NEWCONSENSUS, TorCtl.EVENT_TYPE.NEWDESC, TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW, ], True, ) c.set_option("__LeaveStreamsUnattached", "1") f = c.get_option("FetchUselessDescriptors")[0][1] c.set_option("FetchUselessDescriptors", "1") atexit.register(cleanup, *(c, f)) return (c, h)
def stream_bw_event(self, s): """ Record the timestamp of the last stream_bw event to any stream """ if not s.strm_id in self.streams: plog("WARN", "BW event for unknown stream id: "+str(s.strm_id)) else: self.streams[s.strm_id].bw_timestamp = s.arrived_at PathSupport.PathBuilder.stream_bw_event(self, s)
def cleanup(c, f): plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors=" + f) try: # XXX: Remember __LeaveStreamsUnattached and use saved value! c.set_option("__LeaveStreamsUnattached", "0") c.set_option("FetchUselessDescriptors", f) except TorCtl.TorCtlClosed: pass
def add_rtt(self, rtt): # Compute new current value from the last if self.current_rtt == None: self.current_rtt = rtt else: self.current_rtt = (self.current_rtt * 0.5) + (rtt * 0.5) plog("DEBUG", "Computing new current RTT from " + str(rtt) + " to " + str(self.current_rtt))
def refresh_sorted_list(self): """ Sort the list for their current RTTs """ def notlambda(x): # If not measured yet, return a max value if x.current_rtt == None: return 10 else: return x.current_rtt self.sorted_circs = sort_list(self.circuits.values(), notlambda) plog("DEBUG", "Refreshed sorted list of circuits")
def get_entropy(probs): """ Return the entropy of a given list of probabilities """ # Check if the sum is 1 sum = reduce(lambda x, y: x+y, probs, 0.0) plog("DEBUG", "(Sum of probs is "+str(sum)+")") # Compute the entropy entropy = -reduce(lambda x, y: x+(y*math.log(y,2)), probs, 0.0) return entropy
def get_exit_node(meta): ''' ask metatroller for the last exit used ''' reply = meta.send_command_and_check("GETLASTEXIT") p = re.compile('250 LASTEXIT=[\S]+') m = p.match(reply) exit_node = m.group()[13:] plog('DEBUG', 'Current node: ' + exit_node) return exit_node
def path_is_ok(self, path): """ Check if there is currently a circuit with the given path (Routers) """ if path: for c in self.circuits.values(): if c.path == path: plog("ERROR", "Proposed circuit already exists") return False return True
def cleanup(): s = socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.connect((control_host,control_port)) c = PathSupport.Connection(s) c.authenticate(control_pass) # also launches thread... global FUDValue from TorCtl.TorUtil import plog plog("INFO", "Resetting FetchUselessDescriptors="+FUDValue) c.set_option("FetchUselessDescriptors", FUDValue)
def cbt_cdf(bt_event, x): assert(bt_event.xm > 0) if x < bt_event.xm: x = bt_event.xm ret = 1.0-pow(float(bt_event.xm)/x, bt_event.alpha) if ret < 0 or ret > 1.0: plog("WARN", "Ret: "+str(ret)+" XM: "+str(bt_event.xm)+" alpha: "+str(bt_event.alpha)) assert(0 <= ret and ret <= 1.0) return ret
def cleanup(): s = socket.socket(socket.AF_INET,socket.SOCK_STREAM) s.connect((TorUtil.control_host,TorUtil.control_port)) c = PathSupport.Connection(s) c.authenticate_cookie(file("./tor-data/control_auth_cookie", "r")) global FUDValue from TorCtl.TorUtil import plog plog("INFO", "Resetting FetchUselessDescriptors="+FUDValue) c.set_option("FetchUselessDescriptors", FUDValue)
def start_bw_test(self, circ_id): """ Perform a bandwidth-test on circuit with given circ_id """ plog("INFO", "Starting BW-test on circuit " + str(circ_id)) # Enqueue the circuit self.bw_queue.put(circ_id) # Start the stream-thread (512 KB = 524288) bw_tester = BwTester(1000000) bw_tester.setDaemon(True) bw_tester.start()
def cleanup(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((control_host, control_port)) c = PathSupport.Connection(s) c.authenticate(control_pass) # also launches thread... global FUDValue from TorCtl.TorUtil import plog plog("INFO", "Resetting FetchUselessDescriptors=" + FUDValue) c.set_option("FetchUselessDescriptors", FUDValue)
def cleanup(): s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.connect((TorUtil.control_host, TorUtil.control_port)) c = PathSupport.Connection(s) c.authenticate_cookie(file("./tor-data/control_auth_cookie", "r")) global FUDValue from TorCtl.TorUtil import plog plog("INFO", "Resetting FetchUselessDescriptors=" + FUDValue) c.set_option("FetchUselessDescriptors", FUDValue)
def set_target(self, host, port, max_rtt=0): """ Change the properties for generating paths """ if self.target_host != host or self.target_port != port\ or self.max_rtt != max_rtt: self.target_host = host self.target_port = port self.max_rtt = max_rtt self.up_to_date = False plog("INFO", "Set the target to "+self.target_host+":"+ str(self.target_port))
def __init__(self, filename): self.vote_map = {} try: f = file(filename, "r") f.readline() for line in f.readlines(): vote = Vote(line) self.vote_map[vote.idhex] = vote except IOError: plog("NOTICE", "No previous vote data.")
def update(self): """ Update model with the current list of routers """ nodes = self.graph.nodes() for id in nodes: if not id in self.routers: if id: plog("INFO", "Router with ID " + id + " is not known, deleting node ..") self.delete_node(id) plog("INFO", "Updated model with current router-list")
def log_circuit(self, circ): """ To be called when tests are finished for writing any interesting values to a file before closing circ """ self.testing_logger.append(str(circ.setup_duration) + "\t" + str(circ.bw/1024) + "\t" + str(circ.stats.mean)) line_count = self.testing_logger.get_line_count() if line_count >= num_records: plog("INFO", "Enough records, exiting. (line_count = " + str(line_count) + ")") # TODO: How to kill the main thread from here? sys.exit(1)
def cbt_cdf(bt_event, x): assert (bt_event.xm > 0) if x < bt_event.xm: x = bt_event.xm ret = 1.0 - pow(float(bt_event.xm) / x, bt_event.alpha) if ret < 0 or ret > 1.0: plog( "WARN", "Ret: " + str(ret) + " XM: " + str(bt_event.xm) + " alpha: " + str(bt_event.alpha)) assert (0 <= ret and ret <= 1.0) return ret
def ignore_streams(c,hdlr): for stream in c.get_info("stream-status")['stream-status'].rstrip("\n").split("\n"): m = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream) if m: f = m.groupdict() else: return # no streams s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0) plog("DEBUG", "Ignoring foreign stream: %s" % f['sid']) s.ignored = True hdlr.streams[s.strm_id] = s
def delete_node(self, idhex): """ Delete a router from the model """ if idhex in self.graph: # Delete links first edges = self.graph.edge_boundary(idhex) for e in edges: self.graph.delete_edge(e) # Then remove the node self.graph.delete_node(idhex) plog("INFO", "Deleted node with ID " + idhex + " from the model") self.up_to_date = False
def close_all_circs(self): lines = self.c.sendAndRecv("GETINFO circuit-status\r\n")[0][2] if lines: lines = lines.split("\n") else: return for l in lines: if l: line_parts = l.split(" ") plog("INFO", "Closing aleady built circuit "+str(line_parts[0])) self.live_circs[int(line_parts[0])] = True self.circs[int(line_parts[0])] = True self.c.close_circuit(int(line_parts[0]))
def connect(): """ Return a connection to Tor's control port """ try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((config.get(GENERAL, "control_host"), config.getint(GENERAL, "control_port"))) conn = Connection(sock) conn.authenticate() #conn.debug(file("control.log", "w")) except socket.error, e: plog("ERROR", "Could not connect to Tor process .. running?") sys.exit(-1)
def close_all_circs(self): lines = self.c.sendAndRecv("GETINFO circuit-status\r\n")[0][2] if lines: lines = lines.split("\n") else: return for l in lines: if l: line_parts = l.split(" ") plog("INFO", "Closing aleady built circuit " + str(line_parts[0])) self.live_circs[int(line_parts[0])] = True self.circs[int(line_parts[0])] = True self.c.close_circuit(int(line_parts[0]))
def generate_proposals(self): """ Call visit() on the root-node """ self.update() # Reset list of proposals and prefixes for DFS self.proposals = [] self.prefixes.clear() start = time.time() # Start the search self.visit(None, []) self.up_to_date = True plog("INFO", "Generating " + str(len(self.proposals)) + " proposals took " + str(time.time()-start) + " seconds [max_rtt=" + str(self.max_rtt) + "]")
def ignore_streams(c, hdlr): for stream in c.get_info("stream-status")['stream-status'].rstrip( "\n").split("\n"): m = re.match( "(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})", stream) if m: f = m.groupdict() else: return # no streams s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0) plog("DEBUG", "Ignoring foreign stream: %s" % f['sid']) s.ignored = True hdlr.streams[s.strm_id] = s
def enqueue_circ(self, c): """ Enqueue a circuit for measuring RTT """ if c.built: # Get id of c id = c.circ_id if self.model: # Enqueue every hop path_len = len(c.path) for i in xrange(1, path_len): self.ping_queue.put((id, i)) plog("DEBUG", "Enqueued circuit " + str(id) + " hop " + str(i)) # And for the whole circuit ... self.ping_queue.put((id, None)) plog("DEBUG", "Enqueued circuit " + str(id) + " hop None")
def print_info(self): """ Create a string holding info and the proposals for printing """ out = str(self.graph.info()) for p in self.proposals: out += "\nProposal: " + p.to_string() # Only print them out if there are not too much if len(self.proposals) > 50: plog("INFO", "Currently " + str(len(self.proposals)) + " proposals [max_rtt=" + str(self.max_rtt) + "]! Not printing them out ..") else: print(out) # Log all of them to the file if it exists if self.logfile: self.logfile.write(out)
def handle_bw_test(self, s): """ Handle special streams to measure the bandwidth of circs """ output = [s.event_name, str(s.strm_id), s.status, str(s.circ_id), s.target_host, str(s.target_port)] if s.reason: output.append("REASON=" + s.reason) if s.remote_reason: output.append("REMOTE_REASON=" + s.remote_reason) plog("DEBUG", " ".join(output)) # NEW if s.status == "NEW": stream = Stream(s.strm_id, s.target_host, s.target_port, s.status) self.streams[s.strm_id] = stream # Set next circ_id to stream stream.circ = self.bw_queue.get() try: if stream.circ in self.circuits: circ = self.circuits[stream.circ] if circ.built and not circ.closed: self.c.attach_stream(stream.strm_id, circ.circ_id) else: plog("WARN", "Circuit not built or closed") self.close_stream(s.strm_id, 5) else: # Go to next test if circuit is gone or we get an ErrorReply plog("WARN", "Circuit " + str(circ.circ_id) + " does not exist anymore --> closing stream") # Close stream, XXX: Reason? self.close_stream(s.strm_id, 5) except TorCtl.ErrorReply, e: plog("WARN", "Error attaching stream " + str(stream.strm_id) + " :" + str(e.args)) self.close_stream(s.strm_id, 5)