def cleanup(conn):
""" To be called on exit """
# TODO: Stop other threads and close circuits
plog("INFO", "Cleaning up...")
conn.set_option("__LeaveStreamsUnattached", "0")
conn.set_option("__DisablePredictedCircuits", "0")
conn.close()
def choose_url(percentile):
# TODO: Maybe we don't want to read the file *every* time?
# Maybe once per slice?
# Read in the bw auths file
# here is a fine place to make sure we have bwfiles
try:
f = file("./data/bwfiles", "r")
except IOError:
write_file_list('./data')
lines = []
valid = False
for l in f.readlines():
if l == ".\n":
valid = True
break
pair = l.split()
lines.append((int(pair[0]), pair[1]))
if not valid:
plog("ERROR", "File size list is invalid!")
for (pct, fname) in lines:
if percentile < pct:
return random.choice(urls) + fname
raise PathSupport.NoNodesRemain("No nodes left for url choice!")
def setup_handler(out_dir, cookie_file):
plog("INFO", "Connecting to Tor at " + TorUtil.control_host + ":" + str(TorUtil.control_port))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TorUtil.control_host, TorUtil.control_port))
c = PathSupport.Connection(s)
# c.debug(file(out_dir+"/control.log", "w", buffering=0))
c.authenticate_cookie(file(cookie_file, "r"))
h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector)
# ignore existing streams
ignore_streams(c, h)
c.set_event_handler(h)
# c.set_periodic_timer(2.0, "PULSE")
c.set_events(
[
TorCtl.EVENT_TYPE.STREAM,
TorCtl.EVENT_TYPE.BW,
TorCtl.EVENT_TYPE.NEWCONSENSUS,
TorCtl.EVENT_TYPE.NEWDESC,
TorCtl.EVENT_TYPE.CIRC,
TorCtl.EVENT_TYPE.STREAM_BW,
],
True,
)
atexit.register(cleanup)
return (c, h)
def send_command_and_check(self, line):
self.sock.send(line + '\r\n')
reply = self.readline()
if reply[:3] != '250':
plog('ERROR', reply)
raise MetatrollerException(reply)
return reply
def choose_url(percentile):
# TODO: Maybe we don't want to read the file *every* time?
# Maybe once per slice?
# Read in the bw auths file
# here is a fine place to make sure we have bwfiles
try:
f = file("./data/bwfiles", "r")
except IOError:
write_file_list('./data')
lines = []
valid = False
for l in f.readlines():
if l == ".\n":
valid = True
break
pair = l.split()
lines.append((int(pair[0]), pair[1]))
if not valid:
plog("ERROR", "File size list is invalid!")
for (pct, fname) in lines:
if percentile < pct:
return random.choice(urls) + fname
raise PathSupport.NoNodesRemain("No nodes left for url choice!")
def setup_handler(out_dir, cookie_file):
plog(
'INFO', 'Connecting to Tor at ' + TorUtil.control_host + ":" +
str(TorUtil.control_port))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TorUtil.control_host, TorUtil.control_port))
c = PathSupport.Connection(s)
#c.debug(file(out_dir+"/control.log", "w", buffering=0))
c.authenticate_cookie(file(cookie_file, "r"))
h = BwScanHandler(c,
__selmgr,
strm_selector=PathSupport.SmartSocket.StreamSelector)
# ignore existing streams
ignore_streams(c, h)
c.set_event_handler(h)
#c.set_periodic_timer(2.0, "PULSE")
c.set_events([
TorCtl.EVENT_TYPE.STREAM, TorCtl.EVENT_TYPE.BW,
TorCtl.EVENT_TYPE.NEWCONSENSUS, TorCtl.EVENT_TYPE.NEWDESC,
TorCtl.EVENT_TYPE.CIRC, TorCtl.EVENT_TYPE.STREAM_BW
], True)
atexit.register(cleanup)
return (c, h)
def http_request(address):
""" perform an http GET-request and return 1 for success or 0 for failure """
request = urllib2.Request(address)
request.add_header("User-Agent", user_agent)
try:
reply = urllib2.urlopen(request)
decl_length = reply.info().get("Content-Length")
read_len = len(reply.read())
plog("DEBUG", "Read: " + str(read_len) + " of declared " + str(decl_length))
return 1
except (ValueError, urllib2.URLError):
plog("ERROR", "The http-request address " + address + " is malformed")
return 0
except (IndexError, TypeError):
plog("ERROR", "An error occured while negotiating socks5 with Tor")
return 0
except KeyboardInterrupt:
raise KeyboardInterrupt
except socks.Socks5Error, e:
if e.value[0] == 6:
plog("NOTICE", "Tor timed out our SOCKS stream request.")
else:
plog("ERROR", "An unknown HTTP error occured")
traceback.print_exc()
return 0
def heartbeat_event(self, event):
if len(self.live_circs) < MAX_CIRCUITS:
try:
circ_id = self.c.extend_circuit()
plog("INFO", "Launched circuit: "+str(circ_id))
except TorCtl.ErrorReply, e:
plog("WARN", "Can't extend circuit: "+str(e))
def send_command_and_check(self, line):
self.sock.send(line + '\r\n')
reply = self.readline()
if reply[:3] != '250':
plog('ERROR', reply)
raise MetatrollerException(reply)
return reply
def main(argv):
TorUtil.read_config(argv[1])
(
start_pct,
stop_pct,
nodes_per_slice,
save_every,
circs_per_node,
out_dir,
max_fetch_time,
tor_dir,
sleep_start,
sleep_stop,
min_streams,
pid_file_name,
) = read_config(argv[1])
if pid_file_name:
pidfd = file(pid_file_name, "w")
pidfd.write("%d\n" % os.getpid())
pidfd.close()
try:
(c, hdlr) = setup_handler(out_dir, tor_dir + "/control_auth_cookie")
except Exception, e:
traceback.print_exc()
plog("WARN", "Can't connect to Tor: " + str(e))
def get_guards(c, n):
# Get list of live routers
sorted_rlist = filter(lambda r: not r.down,
c.read_routers(c.get_network_status()))
sorted_rlist.sort(lambda x, y: cmp(y.bw, x.bw))
list_len = len(sorted_rlist)
for i in xrange(list_len): sorted_rlist[i].list_rank = i
guard_rst = PathSupport.FlagsRestriction(["Guard"], [])
if pct_start == 100:
pct_rst = PathSupport.PercentileRestriction(0, pct_start, sorted_rlist)
else:
pct_rst = PathSupport.PercentileRestriction(pct_start, pct_start+PCT_SKIP,
sorted_rlist)
# XXX: Hrmm. UniformGenerator was broken?
guard_gen = PathSupport.ExactUniformGenerator(sorted_rlist,
PathSupport.NodeRestrictionList([guard_rst, pct_rst]))
guard_gen.rewind()
ggen = guard_gen.generate()
# Generate 3 guards
guards = []
for i in xrange(n):
g = ggen.next()
plog("NOTICE", str(pct_start)+"%: Generated guard $"+g.idhex+" with rank "+str(g.list_rank)+"/"+str(list_len)+" "+str(round((100.0*g.list_rank)/list_len, 1))+"% with flags "+str(g.flags))
guards.append(g)
return guards
def http_request(address):
''' perform an http GET-request and return 1 for success or 0 for failure '''
request = urllib2.Request(address)
request.add_header('User-Agent', user_agent)
try:
reply = urllib2.urlopen(request)
decl_length = reply.info().get("Content-Length")
read_len = len(reply.read())
plog("DEBUG",
"Read: " + str(read_len) + " of declared " + str(decl_length))
return 1
except (ValueError, urllib2.URLError):
plog('ERROR', 'The http-request address ' + address + ' is malformed')
return 0
except (IndexError, TypeError):
plog('ERROR', 'An error occured while negotiating socks5 with Tor')
return 0
except KeyboardInterrupt:
raise KeyboardInterrupt
except socks.Socks5Error, e:
if e.value[0] == 6:
plog("NOTICE", "Tor timed out our SOCKS stream request.")
else:
plog('ERROR', 'An unknown HTTP error occured')
traceback.print_exc()
return 0
def __init__(self, line):
# node_id=$DB8C6D8E0D51A42BDDA81A9B8A735B41B2CF95D1 bw=231000 diff=209281 nick=rainbowwarrior measured_at=1319822504
self.idhex = re.search("[\s]*node_id=([\S]+)[\s]*", line).group(1)
self.nick = re.search("[\s]*nick=([\S]+)[\s]*", line).group(1)
self.bw = int(re.search("[\s]+bw=([\S]+)[\s]*", line).group(1))
self.measured_at = int(
re.search("[\s]*measured_at=([\S]+)[\s]*", line).group(1))
try:
self.pid_error = float(
re.search("[\s]*pid_error=([\S]+)[\s]*", line).group(1))
self.pid_error_sum = float(
re.search("[\s]*pid_error_sum=([\S]+)[\s]*", line).group(1))
self.pid_delta = float(
re.search("[\s]*pid_delta=([\S]+)[\s]*", line).group(1))
self.pid_bw = float(
re.search("[\s]*pid_bw=([\S]+)[\s]*", line).group(1))
except:
plog("NOTICE", "No previous PID data.")
self.pid_bw = self.bw
self.pid_error = 0
self.pid_delta = 0
self.pid_error_sum = 0
try:
self.updated_at = int(
re.search("[\s]*updated_at=([\S]+)[\s]*", line).group(1))
except:
plog("INFO",
"No updated_at field for " + self.nick + "=" + self.idhex)
self.updated_at = self.measured_at
def print_circuits(self, list=None):
""" Print out the circuits + some info, optionally pass a (sorted) list """
if list: circs = list
else: circs = self.circuits.values()
plog("INFO", "We have " + str(len(circs)) + " circuits:")
for c in circs:
print("+ " + c.to_string())
def heartbeat_event(self, event):
if len(self.live_circs) < MAX_CIRCUITS:
try:
circ_id = self.c.extend_circuit()
plog("INFO", "Launched circuit: " + str(circ_id))
except TorCtl.ErrorReply, e:
plog("WARN", "Can't extend circuit: " + str(e))
def setup_handler(out_dir, cookie_file):
plog("INFO", "Connecting to Tor at " + TorUtil.control_host + ":" + str(TorUtil.control_port))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TorUtil.control_host, TorUtil.control_port))
c = PathSupport.Connection(s)
# c.debug(file(out_dir+"/control.log", "w", buffering=0))
c.authenticate_cookie(file(cookie_file, "r"))
# f = c.get_option("__LeaveStreamsUnattached")[0]
h = BwScanHandler(c, __selmgr, strm_selector=PathSupport.SmartSocket.StreamSelector)
c.set_event_handler(h)
# c.set_periodic_timer(2.0, "PULSE")
c.set_events(
[
TorCtl.EVENT_TYPE.STREAM,
TorCtl.EVENT_TYPE.BW,
TorCtl.EVENT_TYPE.NEWCONSENSUS,
TorCtl.EVENT_TYPE.NEWDESC,
TorCtl.EVENT_TYPE.CIRC,
TorCtl.EVENT_TYPE.STREAM_BW,
],
True,
)
c.set_option("__LeaveStreamsUnattached", "1")
f = c.get_option("FetchUselessDescriptors")[0][1]
c.set_option("FetchUselessDescriptors", "1")
atexit.register(cleanup, *(c, f))
return (c, h)
def stream_bw_event(self, s):
""" Record the timestamp of the last stream_bw event to any stream """
if not s.strm_id in self.streams:
plog("WARN", "BW event for unknown stream id: "+str(s.strm_id))
else:
self.streams[s.strm_id].bw_timestamp = s.arrived_at
PathSupport.PathBuilder.stream_bw_event(self, s)
def cleanup(c, f):
plog("INFO", "Resetting __LeaveStreamsUnattached=0 and FetchUselessDescriptors=" + f)
try:
# XXX: Remember __LeaveStreamsUnattached and use saved value!
c.set_option("__LeaveStreamsUnattached", "0")
c.set_option("FetchUselessDescriptors", f)
except TorCtl.TorCtlClosed:
pass
def add_rtt(self, rtt):
# Compute new current value from the last
if self.current_rtt == None:
self.current_rtt = rtt
else:
self.current_rtt = (self.current_rtt * 0.5) + (rtt * 0.5)
plog("DEBUG", "Computing new current RTT from " + str(rtt) + " to " +
str(self.current_rtt))
def refresh_sorted_list(self):
""" Sort the list for their current RTTs """
def notlambda(x):
# If not measured yet, return a max value
if x.current_rtt == None: return 10
else: return x.current_rtt
self.sorted_circs = sort_list(self.circuits.values(), notlambda)
plog("DEBUG", "Refreshed sorted list of circuits")
def get_entropy(probs):
""" Return the entropy of a given list of probabilities """
# Check if the sum is 1
sum = reduce(lambda x, y: x+y, probs, 0.0)
plog("DEBUG", "(Sum of probs is "+str(sum)+")")
# Compute the entropy
entropy = -reduce(lambda x, y: x+(y*math.log(y,2)), probs, 0.0)
return entropy
def get_exit_node(meta):
''' ask metatroller for the last exit used '''
reply = meta.send_command_and_check("GETLASTEXIT")
p = re.compile('250 LASTEXIT=[\S]+')
m = p.match(reply)
exit_node = m.group()[13:]
plog('DEBUG', 'Current node: ' + exit_node)
return exit_node
def get_exit_node(meta):
''' ask metatroller for the last exit used '''
reply = meta.send_command_and_check("GETLASTEXIT")
p = re.compile('250 LASTEXIT=[\S]+')
m = p.match(reply)
exit_node = m.group()[13:]
plog('DEBUG', 'Current node: ' + exit_node)
return exit_node
def path_is_ok(self, path):
""" Check if there is currently a circuit with the given path (Routers) """
if path:
for c in self.circuits.values():
if c.path == path:
plog("ERROR", "Proposed circuit already exists")
return False
return True
def cleanup():
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.connect((control_host,control_port))
c = PathSupport.Connection(s)
c.authenticate(control_pass) # also launches thread...
global FUDValue
from TorCtl.TorUtil import plog
plog("INFO", "Resetting FetchUselessDescriptors="+FUDValue)
c.set_option("FetchUselessDescriptors", FUDValue)
def cbt_cdf(bt_event, x):
assert(bt_event.xm > 0)
if x < bt_event.xm:
x = bt_event.xm
ret = 1.0-pow(float(bt_event.xm)/x, bt_event.alpha)
if ret < 0 or ret > 1.0:
plog("WARN", "Ret: "+str(ret)+" XM: "+str(bt_event.xm)+" alpha: "+str(bt_event.alpha))
assert(0 <= ret and ret <= 1.0)
return ret
def cleanup():
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
s.connect((TorUtil.control_host,TorUtil.control_port))
c = PathSupport.Connection(s)
c.authenticate_cookie(file("./tor-data/control_auth_cookie", "r"))
global FUDValue
from TorCtl.TorUtil import plog
plog("INFO", "Resetting FetchUselessDescriptors="+FUDValue)
c.set_option("FetchUselessDescriptors", FUDValue)
def start_bw_test(self, circ_id):
""" Perform a bandwidth-test on circuit with given circ_id """
plog("INFO", "Starting BW-test on circuit " + str(circ_id))
# Enqueue the circuit
self.bw_queue.put(circ_id)
# Start the stream-thread (512 KB = 524288)
bw_tester = BwTester(1000000)
bw_tester.setDaemon(True)
bw_tester.start()
def cleanup():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((control_host, control_port))
c = PathSupport.Connection(s)
c.authenticate(control_pass) # also launches thread...
global FUDValue
from TorCtl.TorUtil import plog
plog("INFO", "Resetting FetchUselessDescriptors=" + FUDValue)
c.set_option("FetchUselessDescriptors", FUDValue)
def cleanup():
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TorUtil.control_host, TorUtil.control_port))
c = PathSupport.Connection(s)
c.authenticate_cookie(file("./tor-data/control_auth_cookie", "r"))
global FUDValue
from TorCtl.TorUtil import plog
plog("INFO", "Resetting FetchUselessDescriptors=" + FUDValue)
c.set_option("FetchUselessDescriptors", FUDValue)
def set_target(self, host, port, max_rtt=0):
""" Change the properties for generating paths """
if self.target_host != host or self.target_port != port\
or self.max_rtt != max_rtt:
self.target_host = host
self.target_port = port
self.max_rtt = max_rtt
self.up_to_date = False
plog("INFO", "Set the target to "+self.target_host+":"+
str(self.target_port))
def __init__(self, filename):
self.vote_map = {}
try:
f = file(filename, "r")
f.readline()
for line in f.readlines():
vote = Vote(line)
self.vote_map[vote.idhex] = vote
except IOError:
plog("NOTICE", "No previous vote data.")
def __init__(self, filename):
self.vote_map = {}
try:
f = file(filename, "r")
f.readline()
for line in f.readlines():
vote = Vote(line)
self.vote_map[vote.idhex] = vote
except IOError:
plog("NOTICE", "No previous vote data.")
def update(self):
""" Update model with the current list of routers """
nodes = self.graph.nodes()
for id in nodes:
if not id in self.routers:
if id:
plog("INFO", "Router with ID " + id +
" is not known, deleting node ..")
self.delete_node(id)
plog("INFO", "Updated model with current router-list")
def log_circuit(self, circ):
""" To be called when tests are finished for writing
any interesting values to a file before closing circ """
self.testing_logger.append(str(circ.setup_duration) + "\t" +
str(circ.bw/1024) + "\t" + str(circ.stats.mean))
line_count = self.testing_logger.get_line_count()
if line_count >= num_records:
plog("INFO", "Enough records, exiting. (line_count = " +
str(line_count) + ")")
# TODO: How to kill the main thread from here?
sys.exit(1)
def cbt_cdf(bt_event, x):
assert (bt_event.xm > 0)
if x < bt_event.xm:
x = bt_event.xm
ret = 1.0 - pow(float(bt_event.xm) / x, bt_event.alpha)
if ret < 0 or ret > 1.0:
plog(
"WARN", "Ret: " + str(ret) + " XM: " + str(bt_event.xm) +
" alpha: " + str(bt_event.alpha))
assert (0 <= ret and ret <= 1.0)
return ret
def ignore_streams(c,hdlr):
for stream in c.get_info("stream-status")['stream-status'].rstrip("\n").split("\n"):
m = re.match("(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",stream)
if m:
f = m.groupdict()
else:
return # no streams
s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0)
plog("DEBUG", "Ignoring foreign stream: %s" % f['sid'])
s.ignored = True
hdlr.streams[s.strm_id] = s
def delete_node(self, idhex):
""" Delete a router from the model """
if idhex in self.graph:
# Delete links first
edges = self.graph.edge_boundary(idhex)
for e in edges:
self.graph.delete_edge(e)
# Then remove the node
self.graph.delete_node(idhex)
plog("INFO", "Deleted node with ID " + idhex + " from the model")
self.up_to_date = False
def close_all_circs(self):
lines = self.c.sendAndRecv("GETINFO circuit-status\r\n")[0][2]
if lines: lines = lines.split("\n")
else: return
for l in lines:
if l:
line_parts = l.split(" ")
plog("INFO", "Closing aleady built circuit "+str(line_parts[0]))
self.live_circs[int(line_parts[0])] = True
self.circs[int(line_parts[0])] = True
self.c.close_circuit(int(line_parts[0]))
def connect():
""" Return a connection to Tor's control port """
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((config.get(GENERAL, "control_host"),
config.getint(GENERAL, "control_port")))
conn = Connection(sock)
conn.authenticate()
#conn.debug(file("control.log", "w"))
except socket.error, e:
plog("ERROR", "Could not connect to Tor process .. running?")
sys.exit(-1)
def close_all_circs(self):
lines = self.c.sendAndRecv("GETINFO circuit-status\r\n")[0][2]
if lines: lines = lines.split("\n")
else: return
for l in lines:
if l:
line_parts = l.split(" ")
plog("INFO",
"Closing aleady built circuit " + str(line_parts[0]))
self.live_circs[int(line_parts[0])] = True
self.circs[int(line_parts[0])] = True
self.c.close_circuit(int(line_parts[0]))
def generate_proposals(self):
""" Call visit() on the root-node """
self.update()
# Reset list of proposals and prefixes for DFS
self.proposals = []
self.prefixes.clear()
start = time.time()
# Start the search
self.visit(None, [])
self.up_to_date = True
plog("INFO", "Generating " + str(len(self.proposals)) +
" proposals took " + str(time.time()-start) +
" seconds [max_rtt=" + str(self.max_rtt) + "]")
def ignore_streams(c, hdlr):
for stream in c.get_info("stream-status")['stream-status'].rstrip(
"\n").split("\n"):
m = re.match(
"(?P<sid>\d*)\s(?P<status>\S*)\s(?P<cid>\d*)\s(?P<host>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(?P<port>\d{1,5})",
stream)
if m:
f = m.groupdict()
else:
return # no streams
s = PathSupport.Stream(int(f['sid']), f['host'], int(f['port']), 0)
plog("DEBUG", "Ignoring foreign stream: %s" % f['sid'])
s.ignored = True
hdlr.streams[s.strm_id] = s
def enqueue_circ(self, c):
""" Enqueue a circuit for measuring RTT """
if c.built:
# Get id of c
id = c.circ_id
if self.model:
# Enqueue every hop
path_len = len(c.path)
for i in xrange(1, path_len):
self.ping_queue.put((id, i))
plog("DEBUG", "Enqueued circuit " + str(id) + " hop " + str(i))
# And for the whole circuit ...
self.ping_queue.put((id, None))
plog("DEBUG", "Enqueued circuit " + str(id) + " hop None")
def print_info(self):
""" Create a string holding info and the proposals for printing """
out = str(self.graph.info())
for p in self.proposals:
out += "\nProposal: " + p.to_string()
# Only print them out if there are not too much
if len(self.proposals) > 50:
plog("INFO", "Currently " + str(len(self.proposals)) +
" proposals [max_rtt=" + str(self.max_rtt) +
"]! Not printing them out ..")
else:
print(out)
# Log all of them to the file if it exists
if self.logfile: self.logfile.write(out)
def handle_bw_test(self, s):
""" Handle special streams to measure the bandwidth of circs """
output = [s.event_name, str(s.strm_id), s.status, str(s.circ_id),
s.target_host, str(s.target_port)]
if s.reason: output.append("REASON=" + s.reason)
if s.remote_reason: output.append("REMOTE_REASON=" + s.remote_reason)
plog("DEBUG", " ".join(output))
# NEW
if s.status == "NEW":
stream = Stream(s.strm_id, s.target_host, s.target_port, s.status)
self.streams[s.strm_id] = stream
# Set next circ_id to stream
stream.circ = self.bw_queue.get()
try:
if stream.circ in self.circuits:
circ = self.circuits[stream.circ]
if circ.built and not circ.closed:
self.c.attach_stream(stream.strm_id, circ.circ_id)
else:
plog("WARN", "Circuit not built or closed")
self.close_stream(s.strm_id, 5)
else:
# Go to next test if circuit is gone or we get an ErrorReply
plog("WARN", "Circuit " + str(circ.circ_id) +
" does not exist anymore --> closing stream")
# Close stream, XXX: Reason?
self.close_stream(s.strm_id, 5)
except TorCtl.ErrorReply, e:
plog("WARN", "Error attaching stream " + str(stream.strm_id) +
" :" + str(e.args))
self.close_stream(s.strm_id, 5)
