def __init__(self, control, errback=None):
self._setup = False
self.attrs = {}
'''After _setup is True, these are all we show as attributes.'''
self.protocol = ITorControlProtocol(control)
self.errback = errback
self.post_bootstrap = defer.Deferred()
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap)
else:
self.bootstrap()
def __init__(self, control=None):
self.config = {}
'''Current configuration, by keys.'''
if control is None:
self.protocol = None
self.__dict__['_slutty_'] = None
self.__dict__['config']['HiddenServices'] = _ListWrapper([], functools.partial(self.mark_unsaved, 'HiddenServices'))
else:
self.protocol = ITorControlProtocol(control)
self.unsaved = {}
'''Configuration that has been changed since last save().'''
self.parsers = {}
'''Instances of the parser classes, subclasses of TorConfigType'''
self.post_bootstrap = defer.Deferred()
if self.protocol:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap).addErrback(log.err)
else:
self.bootstrap()
else:
self.post_bootstrap.callback(self)
self.__dict__['_setup_'] = None
def __init__(self, control=None):
if control is None:
self.protocol = None
self.__dict__['_slutty_'] = None
else:
self.protocol = ITorControlProtocol(control)
self.config = {}
'''Current configuration, by keys.'''
self.unsaved = {}
'''Configuration that has been changed since last save().'''
self.parsers = {}
'''Instances of the parser classes, subclasses of TorConfigType'''
self.post_bootstrap = defer.Deferred()
if self.protocol:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap).addErrback(log.err)
else:
self.bootstrap()
else:
self.post_bootstrap.callback(self)
self.__dict__['_setup_'] = None
def __init__(self, control, errback=None):
self._setup = False
self.attrs = {}
'''After _setup is True, these are all we show as attributes.'''
self.protocol = ITorControlProtocol(control)
self.errback = errback
self.post_bootstrap = defer.Deferred()
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap)
else:
self.bootstrap()
def __init__(self, control=None):
self.config = {}
'''Current configuration, by keys.'''
if control is None:
self._protocol = None
self.__dict__['_accept_all_'] = None
else:
self._protocol = ITorControlProtocol(control)
self.unsaved = OrderedDict()
'''Configuration that has been changed since last save().'''
self.parsers = {}
'''Instances of the parser classes, subclasses of TorConfigType'''
self.list_parsers = set(['hiddenservices'])
'''All the names (keys from .parsers) that are a List of something.'''
# during bootstrapping we decide whether we support the
# following features. A thing goes in here if TorConfig
# behaves differently depending upon whether it shows up in
# "GETINFO config/names"
self._supports = dict(
HiddenServiceDirGroupReadable=False
)
self._defaults = dict()
self.post_bootstrap = defer.Deferred()
if self.protocol:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(
self.bootstrap).addErrback(self.post_bootstrap.errback)
else:
self.bootstrap()
else:
self.do_post_bootstrap(self)
self.__dict__['_setup_'] = None
def __init__(self, protocol, bootstrap=True, write_state_diagram=False):
self.protocol = ITorControlProtocol(protocol)
## fixme could use protocol.on_disconnect to re-connect; see issue #3
## could override these to get your own Circuit/Stream subclasses
## to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self.attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
self.circuits = {} # keys on id (integer)
self.streams = {} # keys on id (integer)
self.routers = {} # keys by hexid (string) and by unique names
self.routers_by_name = {} # keys on name, value always list (many duplicate "Unnamed" routers, for example)
self.guards = {} # potentially-usable as entry guards, I think? (any router with 'Guard' flag)
self.entry_guards = {} # from GETINFO entry-guards, our current entry guards
self.unusable_entry_guards = [] # list of entry guards we didn't parse out
self.authorities = {} # keys by name
self.cleanup = None # see set_attacher
class die(object):
__name__ = 'die' # FIXME? just to ease spagetti.py:82's pain
def __init__(self, msg):
self.msg = msg
def __call__(self, *args):
raise RuntimeError(self.msg % tuple(args))
def nothing(*args):
pass
waiting_r = State("waiting_r")
waiting_w = State("waiting_w")
waiting_p = State("waiting_p")
waiting_s = State("waiting_s")
def ignorable_line(x):
return x.strip() == '.' or x.strip() == 'OK' or x[:3] == 'ns/' or x.strip() == ''
waiting_r.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_r.add_transition(Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin))
## FIXME use better method/func than die!!
waiting_r.add_transition(Transition(waiting_r, lambda x: x[:2] != 'r ', die('Expected "r " while parsing routers not "%s"')))
waiting_s.add_transition(Transition(waiting_w, lambda x: x[:2] == 's ', self._router_flags))
waiting_s.add_transition(Transition(waiting_s, lambda x: x[:2] == 'a ', self._router_address))
waiting_s.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_s.add_transition(Transition(waiting_r, lambda x: x[:2] != 's ' and x[:2] != 'a ', die('Expected "s " while parsing routers not "%s"')))
waiting_s.add_transition(Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_w.add_transition(Transition(waiting_p, lambda x: x[:2] == 'w ', self._router_bandwidth))
waiting_w.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_w.add_transition(Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin)) # "w" lines are optional
waiting_w.add_transition(Transition(waiting_r, lambda x: x[:2] != 'w ', die('Expected "w " while parsing routers not "%s"')))
waiting_w.add_transition(Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_p.add_transition(Transition(waiting_r, lambda x: x[:2] == 'p ', self._router_policy))
waiting_p.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_p.add_transition(Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin)) # "p" lines are optional
waiting_p.add_transition(Transition(waiting_r, lambda x: x[:2] != 'p ', die('Expected "p " while parsing routers not "%s"')))
waiting_p.add_transition(Transition(waiting_r, lambda x: x.strip() == '.', nothing))
self._network_status_parser = FSM([waiting_r, waiting_s, waiting_w, waiting_p])
if write_state_diagram:
with open('routerfsm.dot', 'w') as fsmfile:
fsmfile.write(self._network_status_parser.dotty())
self.post_bootstrap = defer.Deferred()
if bootstrap:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap).addErrback(self.post_bootstrap.errback)
else:
self._bootstrap()
class TorState(object):
"""
This tracks the current state of Tor using a TorControlProtocol.
On setup it first queries the initial state of streams and
circuits. It then asks for updates via the listeners. It requires
an ITorControlProtocol instance. The control protocol doesn't need
to be bootstrapped yet. The Deferred .post_boostrap is driggered
when the TorState instance is fully ready to go. The easiest way
is to use the helper method
:func:`txtorcon.build_tor_connection`. For details, see the
implementation of that.
You may add an :class:`txtorcon.interface.IStreamAttacher` to
provide a custom mapping for Strams to Circuits (by default Tor
picks by itself).
This is also a good example of the various listeners, and acts as
an :class:`txtorcon.interface.ICircuitContainer` and
:class:`txtorcon.interface.IRouterContainer`.
"""
implements(ICircuitListener, ICircuitContainer, IRouterContainer,
IStreamListener)
def __init__(self, protocol, bootstrap=True, write_state_diagram=False):
self.protocol = ITorControlProtocol(protocol)
## fixme could use protocol.on_disconnect to re-connect; see issue #3
## could override these to get your own Circuit/Stream subclasses
## to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self.attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
self.circuits = {} # keys on id (integer)
self.streams = {} # keys on id (integer)
self.routers = {} # keys by hexid (string) and by unique names
self.routers_by_name = {} # keys on name, value always list (many duplicate "Unnamed" routers, for example)
self.guards = {} # potentially-usable as entry guards, I think? (any router with 'Guard' flag)
self.entry_guards = {} # from GETINFO entry-guards, our current entry guards
self.unusable_entry_guards = [] # list of entry guards we didn't parse out
self.authorities = {} # keys by name
self.cleanup = None # see set_attacher
class die(object):
__name__ = 'die' # FIXME? just to ease spagetti.py:82's pain
def __init__(self, msg):
self.msg = msg
def __call__(self, *args):
raise RuntimeError(self.msg % tuple(args))
def nothing(*args):
pass
waiting_r = State("waiting_r")
waiting_w = State("waiting_w")
waiting_p = State("waiting_p")
waiting_s = State("waiting_s")
def ignorable_line(x):
return x.strip() == '.' or x.strip() == 'OK' or x[:3] == 'ns/' or x.strip() == ''
waiting_r.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_r.add_transition(Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin))
## FIXME use better method/func than die!!
waiting_r.add_transition(Transition(waiting_r, lambda x: x[:2] != 'r ', die('Expected "r " while parsing routers not "%s"')))
waiting_s.add_transition(Transition(waiting_w, lambda x: x[:2] == 's ', self._router_flags))
waiting_s.add_transition(Transition(waiting_s, lambda x: x[:2] == 'a ', self._router_address))
waiting_s.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_s.add_transition(Transition(waiting_r, lambda x: x[:2] != 's ' and x[:2] != 'a ', die('Expected "s " while parsing routers not "%s"')))
waiting_s.add_transition(Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_w.add_transition(Transition(waiting_p, lambda x: x[:2] == 'w ', self._router_bandwidth))
waiting_w.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_w.add_transition(Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin)) # "w" lines are optional
waiting_w.add_transition(Transition(waiting_r, lambda x: x[:2] != 'w ', die('Expected "w " while parsing routers not "%s"')))
waiting_w.add_transition(Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_p.add_transition(Transition(waiting_r, lambda x: x[:2] == 'p ', self._router_policy))
waiting_p.add_transition(Transition(waiting_r, ignorable_line, nothing))
waiting_p.add_transition(Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin)) # "p" lines are optional
waiting_p.add_transition(Transition(waiting_r, lambda x: x[:2] != 'p ', die('Expected "p " while parsing routers not "%s"')))
waiting_p.add_transition(Transition(waiting_r, lambda x: x.strip() == '.', nothing))
self._network_status_parser = FSM([waiting_r, waiting_s, waiting_w, waiting_p])
if write_state_diagram:
with open('routerfsm.dot', 'w') as fsmfile:
fsmfile.write(self._network_status_parser.dotty())
self.post_bootstrap = defer.Deferred()
if bootstrap:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap).addErrback(self.post_bootstrap.errback)
else:
self._bootstrap()
def _router_begin(self, data):
args = data.split()
self._router = Router(self.protocol)
self._router.from_consensus = True
self._router.update(args[1], # nickname
args[2], # idhash
args[3], # orhash
datetime.datetime.strptime(args[4] + args[5], '%Y-%m-%f%H:%M:%S'),
args[6], # ip address
args[7], # ORPort
args[8]) # DirPort
if self._router.id_hex in self.routers:
## FIXME should I do an update() on this one??
self._router = self.routers[self._router.id_hex]
return
if self._router.name in self.routers_by_name:
self.routers_by_name[self._router.name].append(self._router)
else:
self.routers_by_name[self._router.name] = [self._router]
if self._router.name in self.routers:
self.routers[self._router.name] = None
else:
self.routers[self._router.name] = self._router
self.routers[self._router.id_hex] = self._router
def _router_flags(self, data):
args = data.split()
self._router.flags = args[1:]
if 'guard' in self._router.flags:
self.guards[self._router.id_hex] = self._router
if 'authority' in self._router.flags:
self.authorities[self._router.name] = self._router
def _router_address(self, data):
"""only for IPv6 addresses"""
self._router.ip_v6.append(data.split()[1].strip())
def _router_bandwidth(self, data):
args = data.split()
self._router.bandwidth = int(args[1].split('=')[1])
def _router_policy(self, data):
args = data.split()
self._router.policy = args[1:]
self._router = None
@defer.inlineCallbacks
def _bootstrap(self, arg=None):
"This takes an arg so we can use it as a callback (see __init__)."
## update list of routers (must be before we do the
## circuit-status) note that we're feeding each line
## incrementally to a state-machine called
## _network_status_parser, set up in constructor. "ns" should
## be the empty string, but we call _update_network_status for
## the de-duplication of named routers
ns = yield self.protocol.get_info_incremental('ns/all',
self._network_status_parser.process)
self._update_network_status(ns)
## update list of existing circuits
cs = yield self.protocol.get_info_raw('circuit-status')
self._circuit_status(cs)
## update list of streams
ss = yield self.protocol.get_info_raw('stream-status')
self._stream_status(ss)
## update list of existing address-maps
key = 'address-mappings/all'
am = yield self.protocol.get_info_raw(key)
## strip addressmappsings/all= and OK\n from raw data
am = am[len(key) + 1:]
if am.strip() != 'OK':
for line in am.split('\n')[:-1]:
if len(line.strip()) == 0:
continue # FIXME
self.addrmap.update(line)
self._add_events()
entries = yield self.protocol.get_info_raw("entry-guards")
for line in entries.split('\n')[1:]:
if len(line.strip()) == 0 or line.strip() == 'OK':
continue
args = line.split()
(name, status) = args[:2]
name = name[:41]
## this is sometimes redundant, as a missing entry guard
## usually means it won't be in our list of routers right
## now, but just being on the safe side
if status.lower() != 'up':
self.unusable_entry_guards.append(line)
continue
try:
self.entry_guards[name] = self.router_from_id(name)
except KeyError:
self.unusable_entry_guards.append(line)
## in case process/pid doesn't exist and we don't know the PID
## because we own it, we just leave it as 0 (previously
## guessed using psutil, but that only works if there's
## exactly one tor running anyway)
try:
pid = yield self.protocol.get_info_raw("process/pid")
except TorProtocolError:
pid = None
self.tor_pid = 0
if pid:
try:
pid = parse_keywords(pid)['process/pid']
self.tor_pid = int(pid)
except KeyError:
self.tor_pid = 0
elif self.protocol.is_owned:
self.tor_pid = self.protocol.is_owned
self.post_bootstrap.callback(self)
self.post_boostrap = None
def undo_attacher(self):
"""
Shouldn't Tor handle this by turning this back to 0 if the
controller that twiddled it disconnects?
"""
return self.protocol.set_conf("__LeaveStreamsUnattached", 0)
def set_attacher(self, attacher, myreactor):
"""
Provide an :class:`txtorcon.interface.IStreamAttacher to
associate streams to circuits. This won't get turned on until
after bootstrapping is completed. ('__LeaveStreamsUnattached'
needs to be set to '1' and the existing circuits list needs to
be populated).
"""
react = IReactorCore(myreactor)
if attacher:
self.attacher = IStreamAttacher(attacher)
else:
self.attacher = None
if self.attacher is None:
self.undo_attacher()
if self.cleanup:
react.removeSystemEventTrigger(self.cleanup)
self.cleanup = None
else:
self.protocol.set_conf("__LeaveStreamsUnattached", "1")
self.cleanup = react.addSystemEventTrigger('before', 'shutdown',
self.undo_attacher)
return None
stream_close_reasons = {
'REASON_MISC': 1, # (catch-all for unlisted reasons)
'REASON_RESOLVEFAILED': 2, # (couldn't look up hostname)
'REASON_CONNECTREFUSED': 3, # (remote host refused connection) [*]
'REASON_EXITPOLICY': 4, # (OR refuses to connect to host or port)
'REASON_DESTROY': 5, # (Circuit is being destroyed)
'REASON_DONE': 6, # (Anonymized TCP connection was closed)
'REASON_TIMEOUT': 7, # (Connection timed out, or OR timed out while connecting)
'REASON_NOROUTE': 8, # (Routing error while attempting to contact destination)
'REASON_HIBERNATING': 9, # (OR is temporarily hibernating)
'REASON_INTERNAL': 10, # (Internal error at the OR)
'REASON_RESOURCELIMIT': 11, # (OR has no resources to fulfill request)
'REASON_CONNRESET': 12, # (Connection was unexpectedly reset)
'REASON_TORPROTOCOL': 13, # (Sent when closing connection because of Tor protocol violations.)
'REASON_NOTDIRECTORY': 14} # (Client sent RELAY_BEGIN_DIR to a non-directory relay.)
def close_stream(self, stream, reason='REASON_MISC', **kwargs):
"""
This sends a STREAMCLOSE command, using the specified reason
(either an int or one of the 14 strings in section 6.3 of
tor-spec.txt if the argument is a string). Any kwards are
passed through as flags if they evaluated to true
(e.g. "SomeFlag=True"). Currently there are none that Tor accepts.
"""
if type(stream) != int:
## assume it's a Stream instance
stream = stream.id
try:
reason = int(reason)
except ValueError:
try:
reason = TorState.stream_close_reasons[reason]
except KeyError:
raise ValueError('Unknown stream close reason "%s"' % str(reason))
flags = flags_from_dict(kwargs)
## stream is now an ID no matter what we passed in
cmd = 'CLOSESTREAM %d %d%s' % (stream, reason, flags)
return self.protocol.queue_command(cmd)
def close_circuit(self, circid, **kwargs):
"""
This sends a CLOSECIRCUIT command, using any keyword arguments
passed as the Flags (currently, that is just 'IfUnused' which
means to only close the circuit when it is no longer used by
any streams).
:return: a Deferred which callbacks with the result of queuing
the command to Tor (usually "OK"). If you want to instead know
when the circuit is actually-gone, see :meth:`Circuit.close
<txtorcon.circuit.Circuit.close>`
"""
if type(circid) != int:
## assume it's a Circuit instance
circid = circid.id
flags = flags_from_dict(kwargs)
return self.protocol.queue_command('CLOSECIRCUIT %s%s' % (circid, flags))
def add_circuit_listener(self, icircuitlistener):
listen = ICircuitListener(icircuitlistener)
for circ in self.circuits.values():
circ.listen(listen)
self.circuit_listeners.append(listen)
def add_stream_listener(self, istreamlistener):
listen = IStreamListener(istreamlistener)
for stream in self.streams.values():
stream.listen(listen)
self.stream_listeners.append(listen)
def _find_circuit_after_extend(self, x):
ex, circ_id = x.split()
if ex != 'EXTENDED':
raise RuntimeError('Expected EXTENDED, got "%s"' % x)
circ_id = int(circ_id)
circ = self._maybe_create_circuit(circ_id)
circ.update([str(circ_id), 'EXTENDED'])
return circ
def build_circuit(self, routers=None):
"""
Builds a circuit consisting of exactly the routers specified,
in order. This issues an EXTENDCIRCUIT call to Tor with all
the routers specified.
:param routers: a list of Router instances which is the path
desired. A warming is issued if the first one isn't in
self.entry_guards To allow Tor to choose the routers
itself, pass None (the default) for routers.
:return:
A Deferred that will callback with a Circuit instance
(with the .id member being valid, and probably nothing
else).
"""
if routers is None or routers == []:
cmd = "EXTENDCIRCUIT 0"
else:
if routers[0] not in self.entry_guards.values():
warnings.warn("Building a circuit not starting with a guard: %s" % (str(routers),), RuntimeWarning)
cmd = "EXTENDCIRCUIT 0 "
first = True
for router in routers:
if first:
first = False
else:
cmd += ','
if isinstance(router, types.StringType) and len(router) == 40 and hashFromHexId(router):
cmd += router
else:
cmd += router.id_hex[1:]
d = self.protocol.queue_command(cmd)
d.addCallback(self._find_circuit_after_extend)
return d
DO_NOT_ATTACH = object()
def _maybe_attach(self, stream):
"""
If we've got a custom stream-attachment instance (see
set_attacher) this will ask it for the appropriate
circuit. Note that we ignore .exit URIs and let Tor deal with
those (by passing circuit ID 0).
The stream attacher is allowed to return a Deferred which will
callback with the desired circuit.
You may return the special object DO_NOT_ATTACH which will
cause the circuit attacher to simply ignore the stream
(neither attaching it, nor telling Tor to attach it).
"""
if self.attacher:
if stream.target_host is not None and '.exit' in stream.target_host:
## we want to totally ignore .exit URIs as these are
## used to specify a particular exit node, and trying
## to do STREAMATTACH on them will fail with an error
## from Tor anyway.
txtorlog.msg("ignore attacher:", stream)
return
circ = IStreamAttacher(self.attacher).attach_stream(stream, self.circuits)
if circ is self.DO_NOT_ATTACH:
return
if circ is None:
self.protocol.queue_command("ATTACHSTREAM %d 0" % stream.id)
else:
if isinstance(circ, defer.Deferred):
class IssueStreamAttach:
def __init__(self, state, streamid):
self.stream_id = streamid
self.state = state
def __call__(self, arg):
circid = arg.id
self.state.protocol.queue_command("ATTACHSTREAM %d %d" % (self.stream_id, circid))
circ.addCallback(IssueStreamAttach(self, stream.id)).addErrback(log.err)
else:
if circ.id not in self.circuits:
raise RuntimeError("Attacher returned a circuit unknown to me.")
if circ.state != 'BUILT':
raise RuntimeError("Can only attach to BUILT circuits; %d is in %s." % (circ.id, circ.state))
self.protocol.queue_command("ATTACHSTREAM %d %d" % (stream.id, circ.id))
def _circuit_status(self, data):
"""Used internally as a callback for updating Circuit information"""
data = data[len('circuit-status='):].split('\n')[:-1]
## sometimes there's a newline after circuit-status= and
## sometimes not, so we get rid of it
if len(data) and len(data[0].strip()) == 0:
data = data[1:]
for line in data:
self._circuit_update(line)
def _stream_status(self, data):
"Used internally as a callback for updating Stream information"
# there's a slight issue with a single-stream vs >= 2 streams,
# in that in the latter case we have a line by itself with
# "stream-status=" on it followed by the streams EXCEPT in the
# single-stream case which has "stream-status=123 blahblah"
# (i.e. the key + value on one line)
lines = data.split('\n')[:-1]
if len(lines) == 1:
d = lines[0][len('stream-status='):]
# if there are actually 0 streams, then there's nothing
# left to parse
if len(d):
self._stream_update(d)
else:
[self._stream_update(line) for line in lines[1:]]
def _update_network_status(self, data):
"""
Used internally as a callback for updating Router information
from NS and NEWCONSENSUS events.
"""
for line in data.split('\n'):
self._network_status_parser.process(line)
txtorlog.msg(len(self.routers_by_name), "named routers found.")
## remove any names we added that turned out to have dups
for (k, v) in self.routers.items():
if v is None:
txtorlog.msg(len(self.routers_by_name[k]), "dups:", k)
del self.routers[k]
txtorlog.msg(len(self.guards), "GUARDs")
def _maybe_create_circuit(self, circ_id):
if circ_id not in self.circuits:
c = self.circuit_factory(self)
c.listen(self)
[c.listen(x) for x in self.circuit_listeners]
else:
c = self.circuits[circ_id]
return c
def _circuit_update(self, line):
"""
Used internally as a callback to update Circuit information
from CIRC events.
"""
#print "circuit_update",line
args = line.split()
circ_id = int(args[0])
c = self._maybe_create_circuit(circ_id)
c.update(args)
def _stream_update(self, line):
"""
Used internally as a callback to update Stream information
from STREAM events.
"""
#print "stream_update",line
if line.strip() == 'stream-status=':
## this happens if there are no active streams
return
args = line.split()
assert len(args) >= 3
stream_id = int(args[0])
wasnew = False
if stream_id not in self.streams:
stream = self.stream_factory(self)
self.streams[stream_id] = stream
stream.listen(self)
[stream.listen(x) for x in self.stream_listeners]
wasnew = True
self.streams[stream_id].update(args)
## if the update closed the stream, it won't be in our list
## anymore. FIXME: how can we ever hit such a case as the
## first update being a CLOSE?
if wasnew and stream_id in self.streams:
self._maybe_attach(self.streams[stream_id])
def _addr_map(self, addr):
"Internal callback to update DNS cache. Listens to ADDRMAP."
txtorlog.msg(" --> addr_map", addr)
self.addrmap.update(addr)
event_map = {'STREAM': _stream_update,
'CIRC': _circuit_update,
'NS': _update_network_status,
'NEWCONSENSUS': _update_network_status,
'ADDRMAP': _addr_map}
"""event_map used by add_events to map event_name -> unbound method"""
@defer.inlineCallbacks
def _add_events(self):
"""
Add listeners for all the events the controller is interested in.
"""
for (event, func) in self.event_map.items():
## the map contains unbound methods, so we bind them
## to self so they call the right thing
yield self.protocol.add_event_listener(event, types.MethodType(func, self, TorState))
## ICircuitContainer
def find_circuit(self, circid):
"ICircuitContainer API"
return self.circuits[circid]
## IRouterContainer
def router_from_id(self, routerid):
"""IRouterContainer API"""
try:
return self.routers[routerid[:41]]
except KeyError:
if routerid[0] != '$':
raise # just re-raise the KeyError
router = Router(self.protocol)
idhash = routerid[1:41]
nick = ''
is_named = False
if len(routerid) > 41:
nick = routerid[42:]
is_named = routerid[41] == '='
router.update(nick, hashFromHexId(idhash), '0' * 27, 'unknown',
'unknown', '0', '0')
router.name_is_unique = is_named
self.routers[router.id_hex] = router
return router
## implement IStreamListener
def stream_new(self, stream):
"IStreamListener: a new stream has been created"
txtorlog.msg("stream_new", stream)
def stream_succeeded(self, stream):
"IStreamListener: stream has succeeded"
txtorlog.msg("stream_succeeded", stream)
def stream_attach(self, stream, circuit):
"""
IStreamListener: the stream has been attached to a circuit. It
seems you get an attach to None followed by an attach to real
circuit fairly frequently. Perhaps related to __LeaveStreamsUnattached?
"""
txtorlog.msg("stream_attach", stream.id,
stream.target_host, " -> ", circuit)
def stream_detach(self, stream, **kw):
"""
IStreamListener
"""
txtorlog.msg("stream_detach", stream.id)
def stream_closed(self, stream, **kw):
"""
IStreamListener: stream has been closed (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_closed", stream.id)
del self.streams[stream.id]
def stream_failed(self, stream, **kw):
"""
IStreamListener: stream failed for some reason (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_failed", stream.id)
del self.streams[stream.id]
## implement ICircuitListener
def circuit_launched(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_launched", circuit)
self.circuits[circuit.id] = circuit
def circuit_extend(self, circuit, router):
"ICircuitListener API"
txtorlog.msg("circuit_extend:", circuit.id, router)
def circuit_built(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_built:", circuit.id,
"->".join("%s.%s" % (x.name, x.location.countrycode) for x in circuit.path),
circuit.streams)
def circuit_new(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_new:", circuit.id)
self.circuits[circuit.id] = circuit
def circuit_destroy(self, circuit):
"Used by circuit_closed and circuit_failed (below)"
txtorlog.msg("circuit_destroy:", circuit.id)
del self.circuits[circuit.id]
def circuit_closed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_closed", circuit)
self.circuit_destroy(circuit)
def circuit_failed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_failed", circuit, str(kw))
self.circuit_destroy(circuit)
def __init__(self, protocol, bootstrap=True):
self.protocol = ITorControlProtocol(protocol)
# fixme could use protocol.on_disconnect to re-connect; see issue #3
# could override these to get your own Circuit/Stream subclasses
# to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self._attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
#: keys on id (integer)
self.circuits = {}
#: keys on id (integer)
self.streams = {}
#: list of unique routers
self.all_routers = set()
#: keys by hexid (string) and by unique names
self.routers = {}
self._old_routers = {}
#: keys on name, value always list (many duplicate "Unnamed"
#: routers, for example)
self.routers_by_name = {}
#: keys by hexid (string)
self.routers_by_hash = {}
#: potentially-usable as entry guards, I think? (any router
#: with 'Guard' flag)
self.guards = {}
#: from GETINFO entry-guards, our current entry guards
self.entry_guards = {}
#: list of entry guards we didn't parse out
self.unusable_entry_guards = []
#: keys by name
self.authorities = {}
#: see set_attacher
self._cleanup = None
self._network_status_parser = MicrodescriptorParser(self._create_router)
self.post_bootstrap = defer.Deferred()
if bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap)
self.protocol.post_bootstrap.addErrback(self.post_bootstrap.errback)
class TorState(object):
"""
This tracks the current state of Tor using a TorControlProtocol.
On setup it first queries the initial state of streams and
circuits. It then asks for updates via the listeners. It requires
an ITorControlProtocol instance. The control protocol doesn't need
to be bootstrapped yet. The Deferred .post_boostrap is driggered
when the TorState instance is fully ready to go. The easiest way
is to use the helper method
:func:`txtorcon.build_tor_connection`. For details, see the
implementation of that.
You may add an :class:`txtorcon.interface.IStreamAttacher` to
provide a custom mapping for Strams to Circuits (by default Tor
picks by itself).
This is also a good example of the various listeners, and acts as
an :class:`txtorcon.interface.ICircuitContainer` and
:class:`txtorcon.interface.IRouterContainer`.
:cvar DO_NOT_ATTACH:
Constant to return from an IAttacher indicating you don't want to
attach this stream at all.
"""
@classmethod
def from_protocol(cls, protocol, **kw):
'''
Create a new, boot-strapped TorState from a TorControlProtocol
instance.
:return: a Deferred that fires with a TorState instance
'''
state = TorState(protocol, bootstrap=True)
return state.post_bootstrap
def __init__(self, protocol, bootstrap=True):
self.protocol = ITorControlProtocol(protocol)
# fixme could use protocol.on_disconnect to re-connect; see issue #3
# could override these to get your own Circuit/Stream subclasses
# to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self._attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
#: keys on id (integer)
self.circuits = {}
#: keys on id (integer)
self.streams = {}
#: list of unique routers
self.all_routers = set()
#: keys by hexid (string) and by unique names
self.routers = {}
self._old_routers = {}
#: keys on name, value always list (many duplicate "Unnamed"
#: routers, for example)
self.routers_by_name = {}
#: keys by hexid (string)
self.routers_by_hash = {}
#: potentially-usable as entry guards, I think? (any router
#: with 'Guard' flag)
self.guards = {}
#: from GETINFO entry-guards, our current entry guards
self.entry_guards = {}
#: list of entry guards we didn't parse out
self.unusable_entry_guards = []
#: keys by name
self.authorities = {}
#: see set_attacher
self._cleanup = None
self._network_status_parser = MicrodescriptorParser(self._create_router)
self.post_bootstrap = defer.Deferred()
if bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap)
self.protocol.post_bootstrap.addErrback(self.post_bootstrap.errback)
def _create_router(self, **kw):
id_hex = hexIdFromHash(kw['idhash'])
try:
router = self._old_routers[id_hex]
except KeyError:
router = Router(self.protocol)
self.routers[id_hex] = router
router.from_consensus = True
router.update(
kw['nickname'],
kw['idhash'],
kw['orhash'],
kw['modified'],
kw['ip'],
kw['orport'],
kw['dirport'],
)
router.flags = kw.get('flags', [])
if 'bandwidth' in kw:
router.bandwidth = kw['bandwidth']
if 'ip_v6' in kw:
router.ip_v6.extend(kw['ip_v6'])
if 'guard' in router.flags:
self.guards[router.id_hex] = router
if 'authority' in router.flags:
self.authorities[router.name] = router
if router.name in self.routers:
self.routers[router.name] = None
else:
self.routers[router.name] = router
if router.name in self.routers_by_name:
self.routers_by_name[router.name].append(router)
else:
self.routers_by_name[router.name] = [router]
self.routers[router.id_hex] = router
self.routers_by_hash[router.id_hex] = router
self.all_routers.add(router)
@defer.inlineCallbacks
def _bootstrap(self, arg=None):
"This takes an arg so we can use it as a callback (see __init__)."
# update list of routers (must be before we do the
# circuit-status)
# look out! we're depending on get_info_incremental returning
# *lines*, which isn't documented -- but will be true because
# TorControlProtocol is a LineReceiver...
yield self.protocol.get_info_incremental(
'ns/all',
self._network_status_parser.feed_line,
)
self._network_status_parser.done()
# update list of existing circuits
cs = yield self.protocol.get_info_raw('circuit-status')
self._circuit_status(cs)
# update list of streams
ss = yield self.protocol.get_info_raw('stream-status')
self._stream_status(ss)
# update list of existing address-maps
key = 'address-mappings/all'
am = yield self.protocol.get_info_raw(key)
# strip addressmappsings/all= and OK\n from raw data
am = am[len(key) + 1:]
for line in am.split('\n'):
if len(line.strip()) == 0:
continue # FIXME
self.addrmap.update(line)
yield self._add_events()
entries = yield self.protocol.get_info_raw("entry-guards")
for line in entries.split('\n')[1:]:
if len(line.strip()) == 0 or line.strip() == 'OK':
# XXX does this ever really happen?
continue
args = line.split()
(name, status) = args[:2]
name = name[:41]
# this is sometimes redundant, as a missing entry guard
# usually means it won't be in our list of routers right
# now, but just being on the safe side
if status.lower() != 'up':
self.unusable_entry_guards.append(line)
continue
try:
self.entry_guards[name] = self.router_from_id(name)
except KeyError:
self.unusable_entry_guards.append(line)
# in case process/pid doesn't exist and we don't know the PID
# because we own it, we just leave it as 0 (previously
# guessed using psutil, but that only works if there's
# exactly one tor running anyway)
try:
pid = yield self.protocol.get_info_raw("process/pid")
except TorProtocolError:
pid = None
self.tor_pid = 0
if pid:
try:
pid = parse_keywords(pid)['process/pid']
self.tor_pid = int(pid)
except Exception: # fixme: ValueError and KeyError ..?
self.tor_pid = 0
if not self.tor_pid and self.protocol.is_owned:
self.tor_pid = self.protocol.is_owned
self.post_bootstrap.callback(self)
self.post_boostrap = None
# XXX this should be hidden as _undo_attacher
def undo_attacher(self):
"""
Shouldn't Tor handle this by turning this back to 0 if the
controller that twiddled it disconnects?
"""
return self.protocol.set_conf("__LeaveStreamsUnattached", 0)
def set_attacher(self, attacher, myreactor):
"""
Provide an :class:`txtorcon.interface.IStreamAttacher` to
associate streams to circuits.
You are Strongly Encouraged to **not** use this API directly,
and instead use :meth:`txtorcon.Circuit.stream_via` or
:meth:`txtorcon.Circuit.web_agent` instead. If you do need to
use this API, it's an error if you call either of the other
two methods.
This won't get turned on until after bootstrapping is
completed. ('__LeaveStreamsUnattached' needs to be set to '1'
and the existing circuits list needs to be populated).
"""
react = IReactorCore(myreactor)
if attacher:
if self._attacher is attacher:
return
if self._attacher is not None:
raise RuntimeError(
"set_attacher called but we already have an attacher"
)
self._attacher = IStreamAttacher(attacher)
else:
self._attacher = None
if self._attacher is None:
d = self.undo_attacher()
if self._cleanup:
react.removeSystemEventTrigger(self._cleanup)
self._cleanup = None
else:
d = self.protocol.set_conf("__LeaveStreamsUnattached", "1")
self._cleanup = react.addSystemEventTrigger(
'before', 'shutdown',
self.undo_attacher,
)
return d
# noqa
stream_close_reasons = {
'REASON_MISC': 1, # (catch-all for unlisted reasons)
'REASON_RESOLVEFAILED': 2, # (couldn't look up hostname)
'REASON_CONNECTREFUSED': 3, # (remote host refused connection) [*]
'REASON_EXITPOLICY': 4, # (OR refuses to connect to host or port)
'REASON_DESTROY': 5, # (Circuit is being destroyed)
'REASON_DONE': 6, # (Anonymized TCP connection was closed)
'REASON_TIMEOUT': 7, # (Connection timed out, or OR timed out while connecting)
'REASON_NOROUTE': 8, # (Routing error while attempting to contact destination)
'REASON_HIBERNATING': 9, # (OR is temporarily hibernating)
'REASON_INTERNAL': 10, # (Internal error at the OR)
'REASON_RESOURCELIMIT': 11, # (OR has no resources to fulfill request)
'REASON_CONNRESET': 12, # (Connection was unexpectedly reset)
'REASON_TORPROTOCOL': 13, # (Sent when closing connection because of Tor protocol violations.)
'REASON_NOTDIRECTORY': 14} # (Client sent RELAY_BEGIN_DIR to a non-directory relay.)
def close_stream(self, stream, reason='REASON_MISC', **kwargs):
"""
This sends a STREAMCLOSE command, using the specified reason
(either an int or one of the 14 strings in section 6.3 of
tor-spec.txt if the argument is a string). Any kwards are
passed through as flags if they evaluated to true
(e.g. "SomeFlag=True"). Currently there are none that Tor accepts.
"""
if type(stream) != int:
# assume it's a Stream instance
stream = stream.id
try:
reason = int(reason)
except ValueError:
try:
reason = TorState.stream_close_reasons[reason]
except KeyError:
raise ValueError(
'Unknown stream close reason "%s"' % str(reason)
)
flags = flags_from_dict(kwargs)
# stream is now an ID no matter what we passed in
cmd = 'CLOSESTREAM %d %d%s' % (stream, reason, flags)
return self.protocol.queue_command(cmd)
def close_circuit(self, circid, **kwargs):
"""
This sends a CLOSECIRCUIT command, using any keyword arguments
passed as the Flags (currently, that is just 'IfUnused' which
means to only close the circuit when it is no longer used by
any streams).
:param circid:
Either a circuit-id (int) or a Circuit instance
:return:
a Deferred which callbacks with the result of queuing the
command to Tor (usually "OK"). If you want to instead know
when the circuit is actually-gone, see
:meth:`Circuit.close <txtorcon.circuit.Circuit.close>`
"""
if type(circid) != int:
# assume it's a Circuit instance
circid = circid.id
flags = flags_from_dict(kwargs)
return self.protocol.queue_command(
'CLOSECIRCUIT %s%s' % (circid, flags)
)
def add_circuit_listener(self, icircuitlistener):
"""
Adds a new instance of :class:`txtorcon.interface.ICircuitListener` which
will receive updates for all existing and new circuits.
"""
listen = ICircuitListener(icircuitlistener)
for circ in self.circuits.values():
circ.listen(listen)
self.circuit_listeners.append(listen)
def add_stream_listener(self, istreamlistener):
"""
Adds a new instance of :class:`txtorcon.interface.IStreamListener` which
will receive updates for all existing and new streams.
"""
listen = IStreamListener(istreamlistener)
for stream in self.streams.values():
stream.listen(listen)
self.stream_listeners.append(listen)
def _find_circuit_after_extend(self, x):
ex, circ_id = x.split()
if ex != 'EXTENDED':
raise RuntimeError('Expected EXTENDED, got "%s"' % x)
circ_id = int(circ_id)
circ = self._maybe_create_circuit(circ_id)
circ.update([str(circ_id), 'EXTENDED'])
return circ
def build_circuit(self, routers=None, using_guards=True, purpose=None):
"""
Builds a circuit consisting of exactly the routers specified,
in order. This issues an EXTENDCIRCUIT call to Tor with all
the routers specified.
:param routers: a list of Router instances which is the path
desired. To allow Tor to choose the routers itself, pass
None (the default) for routers.
:param using_guards: A warning is issued if the first router
isn't in self.entry_guards.
:return:
A Deferred that will callback with a Circuit instance
(with the .id member being valid, and probably nothing
else).
"""
if routers is None or routers == []:
cmd = "EXTENDCIRCUIT 0"
else:
if using_guards and routers[0] not in self.entry_guards.values():
warnings.warn(
"Circuit doesn't start with a guard: %s" % routers,
RuntimeWarning
)
cmd = "EXTENDCIRCUIT 0 "
first = True
for router in routers:
if first:
first = False
else:
cmd += ','
# XXX should we really accept bytes here?
if isinstance(router, bytes) and len(router) == 40 \
and hashFromHexId(router):
cmd += router.decode('utf8')
else:
cmd += router.id_hex[1:]
if purpose is not None:
cmd += " purpose={}".format(purpose)
d = self.protocol.queue_command(cmd)
d.addCallback(self._find_circuit_after_extend)
return d
DO_NOT_ATTACH = object()
# @defer.inlineCallbacks (this method is async, be nice to mark it ...)
def _maybe_attach(self, stream):
"""
If we've got a custom stream-attachment instance (see
set_attacher) this will ask it for the appropriate
circuit. Note that we ignore .exit URIs and let Tor deal with
those (by passing circuit ID 0).
The stream attacher is allowed to return a Deferred which will
callback with the desired circuit.
You may return the special object DO_NOT_ATTACH which will
cause the circuit attacher to simply ignore the stream
(neither attaching it, nor telling Tor to attach it).
"""
if self._attacher is None:
return None
if stream.target_host is not None \
and '.exit' in stream.target_host:
# we want to totally ignore .exit URIs as these are
# used to specify a particular exit node, and trying
# to do STREAMATTACH on them will fail with an error
# from Tor anyway.
txtorlog.msg("ignore attacher:", stream)
return
# handle async or sync .attach() the same
circ_d = defer.maybeDeferred(
self._attacher.attach_stream,
stream, self.circuits,
)
circ_d.addCallback(maybe_coroutine)
# actually do the attachment logic; .attach() can return 3 things:
# 1. None: let Tor do whatever it wants
# 2. DO_NOT_ATTACH: don't attach the stream at all
# 3. Circuit instance: attach to the provided circuit
def issue_stream_attach(circ):
txtorlog.msg("circuit:", circ)
if circ is None or circ is TorState.DO_NOT_ATTACH:
# tell Tor to do what it likes
return self.protocol.queue_command(
u"ATTACHSTREAM {} 0".format(stream.id).encode("ascii")
)
else:
# should get a Circuit instance; check it for suitability
if not isinstance(circ, Circuit):
raise RuntimeError(
"IStreamAttacher.attach() must return a Circuit instance "
"(or None or DO_NOT_ATTACH): %s"
)
if circ.id not in self.circuits:
raise RuntimeError(
"Attacher returned a circuit unknown to me."
)
if circ.state != 'BUILT':
raise RuntimeError(
"Can only attach to BUILT circuits; %d is in %s." %
(circ.id, circ.state)
)
# we've got a valid Circuit instance; issue the command
return self.protocol.queue_command(
u"ATTACHSTREAM {} {}".format(stream.id, circ.id).encode("ascii")
)
circ_d.addCallback(issue_stream_attach)
circ_d.addErrback(self._attacher_error)
return circ_d
def _attacher_error(self, fail):
"""
not ideal, but there's not really a good way to let the caller
handler errors :/ since we ultimately call this due to an
async request from Tor. Mostly these errors will be logic or
syntax errors in the caller's code anyway.
tests monkey-patch this to reduce spew
"""
print("Failure while attaching stream:", fail)
return fail
def _circuit_status(self, data):
"""Used internally as a callback for updating Circuit information"""
data = data[len('circuit-status='):].split('\n')
# sometimes there's a newline after circuit-status= and
# sometimes not, so we get rid of it
if len(data) and len(data[0].strip()) == 0:
data = data[1:]
for line in data:
self._circuit_update(line)
def _stream_status(self, data):
"Used internally as a callback for updating Stream information"
# there's a slight issue with a single-stream vs >= 2 streams,
# in that in the latter case we have a line by itself with
# "stream-status=" on it followed by the streams EXCEPT in the
# single-stream case which has "stream-status=123 blahblah"
# (i.e. the key + value on one line)
lines = data.split('\n')
if len(lines) == 1:
d = lines[0][len('stream-status='):]
# if there are actually 0 streams, then there's nothing
# left to parse
if len(d):
self._stream_update(d)
else:
[self._stream_update(line) for line in lines[1:]]
def _update_network_status(self, data):
"""
Used internally as a callback for updating Router information
from NEWCONSENSUS events.
"""
# XXX why are we ever getting this with 0 data?
if len(data):
self._old_routers = self.routers
self.routers = dict()
self.all_routers = set()
self.routers_by_hash = dict()
self.routers_by_name = dict()
for line in data.split('\n'):
self._network_status_parser.feed_line(line)
self._network_status_parser.done()
txtorlog.msg(len(self.routers_by_name), "named routers found.")
# remove any names we added that turned out to have dups
remove_keys = set()
for (k, v) in self.routers.items():
if v is None:
txtorlog.msg(len(self.routers_by_name[k]), "dups:", k)
remove_keys.add(k)
for k in remove_keys:
del self.routers[k]
txtorlog.msg(len(self.guards), "GUARDs")
def _maybe_create_circuit(self, circ_id):
if circ_id not in self.circuits:
c = self.circuit_factory(self)
c.listen(self)
for listener in self.circuit_listeners:
c.listen(listener)
else:
c = self.circuits[circ_id]
return c
def _circuit_update(self, line):
"""
Used internally as a callback to update Circuit information
from CIRC events.
"""
# print("circuit_update", line)
args = line.split()
circ_id = int(args[0])
c = self._maybe_create_circuit(circ_id)
c.update(args)
def _stream_update(self, line):
"""
Used internally as a callback to update Stream information
from STREAM events.
"""
if line.strip() == 'stream-status=':
# this happens if there are no active streams
return
args = line.split()
assert len(args) >= 3
stream_id = int(args[0])
wasnew = False
if stream_id not in self.streams:
stream = self.stream_factory(self, self.addrmap)
self.streams[stream_id] = stream
stream.listen(self)
for x in self.stream_listeners:
stream.listen(x)
wasnew = True
self.streams[stream_id].update(args)
# if the update closed the stream, it won't be in our list
# anymore. FIXME: how can we ever hit such a case as the
# first update being a CLOSE?
if wasnew and stream_id in self.streams:
self._maybe_attach(self.streams[stream_id])
def _addr_map(self, addr):
"Internal callback to update DNS cache. Listens to ADDRMAP."
txtorlog.msg(" --> addr_map", addr)
self.addrmap.update(addr)
event_map = {
'STREAM': '_stream_update',
'CIRC': '_circuit_update',
'NEWCONSENSUS': '_update_network_status',
'ADDRMAP': '_addr_map',
}
@defer.inlineCallbacks
def _add_events(self):
"""
Add listeners for all the events the controller is interested in.
"""
for (event, func_name) in self.event_map.items():
yield self.protocol.add_event_listener(
event,
getattr(self, func_name),
)
# ICircuitContainer
def find_circuit(self, circid):
"ICircuitContainer API"
return self.circuits[circid]
# IRouterContainer
def router_from_id(self, routerid):
"""IRouterContainer API"""
try:
return self.routers[routerid[:41]]
except KeyError:
if routerid[0] != '$':
raise # just re-raise the KeyError
router = Router(self.protocol)
idhash = routerid[1:41]
nick = ''
is_named = False
if len(routerid) > 41:
nick = routerid[42:]
is_named = routerid[41] == '='
router.update(nick, hashFromHexId(idhash), '0' * 27, 'unknown',
'unknown', '0', '0')
router.name_is_unique = is_named
self.routers[router.id_hex] = router
return router
# implement IStreamListener
def stream_new(self, stream):
"IStreamListener: a new stream has been created"
txtorlog.msg("stream_new", stream)
def stream_succeeded(self, stream):
"IStreamListener: stream has succeeded"
txtorlog.msg("stream_succeeded", stream)
def stream_attach(self, stream, circuit):
"""
IStreamListener: the stream has been attached to a circuit. It
seems you get an attach to None followed by an attach to real
circuit fairly frequently. Perhaps related to __LeaveStreamsUnattached?
"""
txtorlog.msg("stream_attach", stream.id,
stream.target_host, " -> ", circuit)
def stream_detach(self, stream, **kw):
"""
IStreamListener
"""
txtorlog.msg("stream_detach", stream.id)
def stream_closed(self, stream, **kw):
"""
IStreamListener: stream has been closed (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_closed", stream.id)
del self.streams[stream.id]
def stream_failed(self, stream, **kw):
"""
IStreamListener: stream failed for some reason (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_failed", stream.id)
del self.streams[stream.id]
# implement ICircuitListener
def circuit_launched(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_launched", circuit)
self.circuits[circuit.id] = circuit
def circuit_extend(self, circuit, router):
"ICircuitListener API"
txtorlog.msg("circuit_extend:", circuit.id, router)
def circuit_built(self, circuit):
"ICircuitListener API"
txtorlog.msg(
"circuit_built:", circuit.id,
"->".join("%s.%s" % (x.name, x.location.countrycode) for x in circuit.path),
circuit.streams
)
def circuit_new(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_new:", circuit.id)
self.circuits[circuit.id] = circuit
def circuit_destroy(self, circuit):
"Used by circuit_closed and circuit_failed (below)"
txtorlog.msg("circuit_destroy:", circuit.id)
circuit._when_built.fire(
Failure(Exception("Destroying circuit; will never hit BUILT"))
)
del self.circuits[circuit.id]
def circuit_closed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_closed", circuit)
circuit._when_built.fire(
Failure(
CircuitBuildClosedError(_extract_reason(kw))
)
)
self.circuit_destroy(circuit)
def circuit_failed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_failed", circuit, str(kw))
circuit._when_built.fire(
Failure(
CircuitBuildFailedError(_extract_reason(kw))
)
)
self.circuit_destroy(circuit)
def __init__(self, protocol, bootstrap=True, write_state_diagram=False):
self.protocol = ITorControlProtocol(protocol)
## fixme could use protocol.on_disconnect to re-connect; see issue #3
## could override these to get your own Circuit/Stream subclasses
## to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self.attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
self.circuits = {} # keys on id (integer)
self.streams = {} # keys on id (integer)
self.routers = {} # keys by hexid (string) and by unique names
self.routers_by_name = {
} # keys on name, value always list (many duplicate "Unnamed" routers, for example)
self.guards = {
} # potentially-usable as entry guards, I think? (any router with 'Guard' flag)
self.entry_guards = {
} # from GETINFO entry-guards, our current entry guards
self.unusable_entry_guards = [
] # list of entry guards we didn't parse out
self.authorities = {} # keys by name
self.cleanup = None # see set_attacher
class die(object):
__name__ = 'die' # FIXME? just to ease spagetti.py:82's pain
def __init__(self, msg):
self.msg = msg
def __call__(self, *args):
raise RuntimeError(self.msg % tuple(args))
def nothing(*args):
pass
waiting_r = State("waiting_r")
waiting_w = State("waiting_w")
waiting_p = State("waiting_p")
waiting_s = State("waiting_s")
def ignorable_line(x):
return x.strip() == '.' or x.strip(
) == 'OK' or x[:3] == 'ns/' or x.strip() == ''
waiting_r.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_r.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin))
## FIXME use better method/func than die!!
waiting_r.add_transition(
Transition(waiting_r, lambda x: x[:2] != 'r ',
die('Expected "r " while parsing routers not "%s"')))
waiting_s.add_transition(
Transition(waiting_w, lambda x: x[:2] == 's ', self._router_flags))
waiting_s.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'a ',
self._router_address))
waiting_s.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_s.add_transition(
Transition(waiting_r, lambda x: x[:2] != 's ' and x[:2] != 'a ',
die('Expected "s " while parsing routers not "%s"')))
waiting_s.add_transition(
Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_w.add_transition(
Transition(waiting_p, lambda x: x[:2] == 'w ',
self._router_bandwidth))
waiting_w.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_w.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'r ',
self._router_begin)) # "w" lines are optional
waiting_w.add_transition(
Transition(waiting_r, lambda x: x[:2] != 'w ',
die('Expected "w " while parsing routers not "%s"')))
waiting_w.add_transition(
Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_p.add_transition(
Transition(waiting_r, lambda x: x[:2] == 'p ',
self._router_policy))
waiting_p.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_p.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'r ',
self._router_begin)) # "p" lines are optional
waiting_p.add_transition(
Transition(waiting_r, lambda x: x[:2] != 'p ',
die('Expected "p " while parsing routers not "%s"')))
waiting_p.add_transition(
Transition(waiting_r, lambda x: x.strip() == '.', nothing))
self._network_status_parser = FSM(
[waiting_r, waiting_s, waiting_w, waiting_p])
if write_state_diagram:
with open('routerfsm.dot', 'w') as fsmfile:
fsmfile.write(self._network_status_parser.dotty())
self.post_bootstrap = defer.Deferred()
if bootstrap:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(
self._bootstrap).addErrback(self.post_bootstrap.errback)
else:
self._bootstrap()
class TorState(object):
"""
This tracks the current state of Tor using a TorControlProtocol.
On setup it first queries the initial state of streams and
circuits. It then asks for updates via the listeners. It requires
an ITorControlProtocol instance. The control protocol doesn't need
to be bootstrapped yet. The Deferred .post_boostrap is driggered
when the TorState instance is fully ready to go. The easiest way
is to use the helper method
:func:`txtorcon.build_tor_connection`. For details, see the
implementation of that.
You may add an :class:`txtorcon.interface.IStreamAttacher` to
provide a custom mapping for Strams to Circuits (by default Tor
picks by itself).
This is also a good example of the various listeners, and acts as
an :class:`txtorcon.interface.ICircuitContainer` and
:class:`txtorcon.interface.IRouterContainer`.
"""
implements(ICircuitListener, ICircuitContainer, IRouterContainer,
IStreamListener)
def __init__(self, protocol, bootstrap=True, write_state_diagram=False):
self.protocol = ITorControlProtocol(protocol)
## fixme could use protocol.on_disconnect to re-connect; see issue #3
## could override these to get your own Circuit/Stream subclasses
## to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self.attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
self.circuits = {} # keys on id (integer)
self.streams = {} # keys on id (integer)
self.routers = {} # keys by hexid (string) and by unique names
self.routers_by_name = {
} # keys on name, value always list (many duplicate "Unnamed" routers, for example)
self.guards = {
} # potentially-usable as entry guards, I think? (any router with 'Guard' flag)
self.entry_guards = {
} # from GETINFO entry-guards, our current entry guards
self.unusable_entry_guards = [
] # list of entry guards we didn't parse out
self.authorities = {} # keys by name
self.cleanup = None # see set_attacher
class die(object):
__name__ = 'die' # FIXME? just to ease spagetti.py:82's pain
def __init__(self, msg):
self.msg = msg
def __call__(self, *args):
raise RuntimeError(self.msg % tuple(args))
def nothing(*args):
pass
waiting_r = State("waiting_r")
waiting_w = State("waiting_w")
waiting_p = State("waiting_p")
waiting_s = State("waiting_s")
def ignorable_line(x):
return x.strip() == '.' or x.strip(
) == 'OK' or x[:3] == 'ns/' or x.strip() == ''
waiting_r.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_r.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'r ', self._router_begin))
## FIXME use better method/func than die!!
waiting_r.add_transition(
Transition(waiting_r, lambda x: x[:2] != 'r ',
die('Expected "r " while parsing routers not "%s"')))
waiting_s.add_transition(
Transition(waiting_w, lambda x: x[:2] == 's ', self._router_flags))
waiting_s.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'a ',
self._router_address))
waiting_s.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_s.add_transition(
Transition(waiting_r, lambda x: x[:2] != 's ' and x[:2] != 'a ',
die('Expected "s " while parsing routers not "%s"')))
waiting_s.add_transition(
Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_w.add_transition(
Transition(waiting_p, lambda x: x[:2] == 'w ',
self._router_bandwidth))
waiting_w.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_w.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'r ',
self._router_begin)) # "w" lines are optional
waiting_w.add_transition(
Transition(waiting_r, lambda x: x[:2] != 'w ',
die('Expected "w " while parsing routers not "%s"')))
waiting_w.add_transition(
Transition(waiting_r, lambda x: x.strip() == '.', nothing))
waiting_p.add_transition(
Transition(waiting_r, lambda x: x[:2] == 'p ',
self._router_policy))
waiting_p.add_transition(Transition(waiting_r, ignorable_line,
nothing))
waiting_p.add_transition(
Transition(waiting_s, lambda x: x[:2] == 'r ',
self._router_begin)) # "p" lines are optional
waiting_p.add_transition(
Transition(waiting_r, lambda x: x[:2] != 'p ',
die('Expected "p " while parsing routers not "%s"')))
waiting_p.add_transition(
Transition(waiting_r, lambda x: x.strip() == '.', nothing))
self._network_status_parser = FSM(
[waiting_r, waiting_s, waiting_w, waiting_p])
if write_state_diagram:
with open('routerfsm.dot', 'w') as fsmfile:
fsmfile.write(self._network_status_parser.dotty())
self.post_bootstrap = defer.Deferred()
if bootstrap:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(
self._bootstrap).addErrback(self.post_bootstrap.errback)
else:
self._bootstrap()
def _router_begin(self, data):
args = data.split()
self._router = Router(self.protocol)
self._router.from_consensus = True
self._router.update(
args[1], # nickname
args[2], # idhash
args[3], # orhash
datetime.datetime.strptime(args[4] + args[5], '%Y-%m-%f%H:%M:%S'),
args[6], # ip address
args[7], # ORPort
args[8]) # DirPort
if self._router.id_hex in self.routers:
## FIXME should I do an update() on this one??
self._router = self.routers[self._router.id_hex]
return
if self._router.name in self.routers_by_name:
self.routers_by_name[self._router.name].append(self._router)
else:
self.routers_by_name[self._router.name] = [self._router]
if self._router.name in self.routers:
self.routers[self._router.name] = None
else:
self.routers[self._router.name] = self._router
self.routers[self._router.id_hex] = self._router
def _router_flags(self, data):
args = data.split()
self._router.flags = args[1:]
if 'guard' in self._router.flags:
self.guards[self._router.id_hex] = self._router
if 'authority' in self._router.flags:
self.authorities[self._router.name] = self._router
def _router_address(self, data):
"""only for IPv6 addresses"""
self._router.ip_v6.append(data.split()[1].strip())
def _router_bandwidth(self, data):
args = data.split()
self._router.bandwidth = int(args[1].split('=')[1])
def _router_policy(self, data):
args = data.split()
self._router.policy = args[1:]
self._router = None
@defer.inlineCallbacks
def _bootstrap(self, arg=None):
"This takes an arg so we can use it as a callback (see __init__)."
## update list of routers (must be before we do the
## circuit-status) note that we're feeding each line
## incrementally to a state-machine called
## _network_status_parser, set up in constructor. "ns" should
## be the empty string, but we call _update_network_status for
## the de-duplication of named routers
ns = yield self.protocol.get_info_incremental(
'ns/all', self._network_status_parser.process)
self._update_network_status(ns)
## update list of existing circuits
cs = yield self.protocol.get_info_raw('circuit-status')
self._circuit_status(cs)
## update list of streams
ss = yield self.protocol.get_info_raw('stream-status')
self._stream_status(ss)
## update list of existing address-maps
key = 'address-mappings/all'
am = yield self.protocol.get_info_raw(key)
## strip addressmappsings/all= and OK\n from raw data
am = am[len(key) + 1:]
for line in am.split('\n'):
if len(line.strip()) == 0:
continue # FIXME
self.addrmap.update(line)
self._add_events()
entries = yield self.protocol.get_info_raw("entry-guards")
for line in entries.split('\n')[1:]:
if len(line.strip()) == 0 or line.strip() == 'OK':
continue
args = line.split()
(name, status) = args[:2]
name = name[:41]
## this is sometimes redundant, as a missing entry guard
## usually means it won't be in our list of routers right
## now, but just being on the safe side
if status.lower() != 'up':
self.unusable_entry_guards.append(line)
continue
try:
self.entry_guards[name] = self.router_from_id(name)
except KeyError:
self.unusable_entry_guards.append(line)
## in case process/pid doesn't exist and we don't know the PID
## because we own it, we just leave it as 0 (previously
## guessed using psutil, but that only works if there's
## exactly one tor running anyway)
try:
pid = yield self.protocol.get_info_raw("process/pid")
except TorProtocolError:
pid = None
self.tor_pid = 0
if pid:
try:
pid = parse_keywords(pid)['process/pid']
self.tor_pid = int(pid)
except KeyError:
self.tor_pid = 0
elif self.protocol.is_owned:
self.tor_pid = self.protocol.is_owned
self.post_bootstrap.callback(self)
self.post_boostrap = None
def undo_attacher(self):
"""
Shouldn't Tor handle this by turning this back to 0 if the
controller that twiddled it disconnects?
"""
return self.protocol.set_conf("__LeaveStreamsUnattached", 0)
def set_attacher(self, attacher, myreactor):
"""
Provide an :class:`txtorcon.interface.IStreamAttacher` to
associate streams to circuits. This won't get turned on until
after bootstrapping is completed. ('__LeaveStreamsUnattached'
needs to be set to '1' and the existing circuits list needs to
be populated).
"""
react = IReactorCore(myreactor)
if attacher:
self.attacher = IStreamAttacher(attacher)
else:
self.attacher = None
if self.attacher is None:
self.undo_attacher()
if self.cleanup:
react.removeSystemEventTrigger(self.cleanup)
self.cleanup = None
else:
self.protocol.set_conf("__LeaveStreamsUnattached", "1")
self.cleanup = react.addSystemEventTrigger('before', 'shutdown',
self.undo_attacher)
return None
stream_close_reasons = {
'REASON_MISC': 1, # (catch-all for unlisted reasons)
'REASON_RESOLVEFAILED': 2, # (couldn't look up hostname)
'REASON_CONNECTREFUSED': 3, # (remote host refused connection) [*]
'REASON_EXITPOLICY': 4, # (OR refuses to connect to host or port)
'REASON_DESTROY': 5, # (Circuit is being destroyed)
'REASON_DONE': 6, # (Anonymized TCP connection was closed)
'REASON_TIMEOUT':
7, # (Connection timed out, or OR timed out while connecting)
'REASON_NOROUTE':
8, # (Routing error while attempting to contact destination)
'REASON_HIBERNATING': 9, # (OR is temporarily hibernating)
'REASON_INTERNAL': 10, # (Internal error at the OR)
'REASON_RESOURCELIMIT': 11, # (OR has no resources to fulfill request)
'REASON_CONNRESET': 12, # (Connection was unexpectedly reset)
'REASON_TORPROTOCOL':
13, # (Sent when closing connection because of Tor protocol violations.)
'REASON_NOTDIRECTORY': 14
} # (Client sent RELAY_BEGIN_DIR to a non-directory relay.)
def close_stream(self, stream, reason='REASON_MISC', **kwargs):
"""
This sends a STREAMCLOSE command, using the specified reason
(either an int or one of the 14 strings in section 6.3 of
tor-spec.txt if the argument is a string). Any kwards are
passed through as flags if they evaluated to true
(e.g. "SomeFlag=True"). Currently there are none that Tor accepts.
"""
if type(stream) != int:
## assume it's a Stream instance
stream = stream.id
try:
reason = int(reason)
except ValueError:
try:
reason = TorState.stream_close_reasons[reason]
except KeyError:
raise ValueError('Unknown stream close reason "%s"' %
str(reason))
flags = flags_from_dict(kwargs)
## stream is now an ID no matter what we passed in
cmd = 'CLOSESTREAM %d %d%s' % (stream, reason, flags)
return self.protocol.queue_command(cmd)
def close_circuit(self, circid, **kwargs):
"""
This sends a CLOSECIRCUIT command, using any keyword arguments
passed as the Flags (currently, that is just 'IfUnused' which
means to only close the circuit when it is no longer used by
any streams).
:param circid:
Either a circuit-id (int) or a Circuit instance
:return:
a Deferred which callbacks with the result of queuing the
command to Tor (usually "OK"). If you want to instead know
when the circuit is actually-gone, see
:meth:`Circuit.close <txtorcon.circuit.Circuit.close>`
"""
if type(circid) != int:
## assume it's a Circuit instance
circid = circid.id
flags = flags_from_dict(kwargs)
return self.protocol.queue_command('CLOSECIRCUIT %s%s' %
(circid, flags))
def add_circuit_listener(self, icircuitlistener):
listen = ICircuitListener(icircuitlistener)
for circ in self.circuits.values():
circ.listen(listen)
self.circuit_listeners.append(listen)
def add_stream_listener(self, istreamlistener):
listen = IStreamListener(istreamlistener)
for stream in self.streams.values():
stream.listen(listen)
self.stream_listeners.append(listen)
def _find_circuit_after_extend(self, x):
ex, circ_id = x.split()
if ex != 'EXTENDED':
raise RuntimeError('Expected EXTENDED, got "%s"' % x)
circ_id = int(circ_id)
circ = self._maybe_create_circuit(circ_id)
circ.update([str(circ_id), 'EXTENDED'])
return circ
def build_circuit(self, routers=None):
"""
Builds a circuit consisting of exactly the routers specified,
in order. This issues an EXTENDCIRCUIT call to Tor with all
the routers specified.
:param routers: a list of Router instances which is the path
desired. A warming is issued if the first one isn't in
self.entry_guards To allow Tor to choose the routers
itself, pass None (the default) for routers.
:return:
A Deferred that will callback with a Circuit instance
(with the .id member being valid, and probably nothing
else).
"""
if routers is None or routers == []:
cmd = "EXTENDCIRCUIT 0"
else:
if routers[0] not in self.entry_guards.values():
warnings.warn(
"Building a circuit not starting with a guard: %s" %
(str(routers), ), RuntimeWarning)
cmd = "EXTENDCIRCUIT 0 "
first = True
for router in routers:
if first:
first = False
else:
cmd += ','
if isinstance(router, types.StringType) and len(
router) == 40 and hashFromHexId(router):
cmd += router
else:
cmd += router.id_hex[1:]
d = self.protocol.queue_command(cmd)
d.addCallback(self._find_circuit_after_extend)
return d
DO_NOT_ATTACH = object()
def _maybe_attach(self, stream):
"""
If we've got a custom stream-attachment instance (see
set_attacher) this will ask it for the appropriate
circuit. Note that we ignore .exit URIs and let Tor deal with
those (by passing circuit ID 0).
The stream attacher is allowed to return a Deferred which will
callback with the desired circuit.
You may return the special object DO_NOT_ATTACH which will
cause the circuit attacher to simply ignore the stream
(neither attaching it, nor telling Tor to attach it).
"""
if self.attacher:
if stream.target_host is not None and '.exit' in stream.target_host:
## we want to totally ignore .exit URIs as these are
## used to specify a particular exit node, and trying
## to do STREAMATTACH on them will fail with an error
## from Tor anyway.
txtorlog.msg("ignore attacher:", stream)
return
circ = IStreamAttacher(self.attacher).attach_stream(
stream, self.circuits)
if circ is self.DO_NOT_ATTACH:
return
if circ is None:
self.protocol.queue_command("ATTACHSTREAM %d 0" % stream.id)
else:
if isinstance(circ, defer.Deferred):
class IssueStreamAttach:
def __init__(self, state, streamid):
self.stream_id = streamid
self.state = state
def __call__(self, arg):
circid = arg.id
self.state.protocol.queue_command(
"ATTACHSTREAM %d %d" %
(self.stream_id, circid))
circ.addCallback(IssueStreamAttach(
self, stream.id)).addErrback(log.err)
else:
if circ.id not in self.circuits:
raise RuntimeError(
"Attacher returned a circuit unknown to me.")
if circ.state != 'BUILT':
raise RuntimeError(
"Can only attach to BUILT circuits; %d is in %s." %
(circ.id, circ.state))
self.protocol.queue_command("ATTACHSTREAM %d %d" %
(stream.id, circ.id))
def _circuit_status(self, data):
"""Used internally as a callback for updating Circuit information"""
data = data[len('circuit-status='):].split('\n')
## sometimes there's a newline after circuit-status= and
## sometimes not, so we get rid of it
if len(data) and len(data[0].strip()) == 0:
data = data[1:]
for line in data:
self._circuit_update(line)
def _stream_status(self, data):
"Used internally as a callback for updating Stream information"
# there's a slight issue with a single-stream vs >= 2 streams,
# in that in the latter case we have a line by itself with
# "stream-status=" on it followed by the streams EXCEPT in the
# single-stream case which has "stream-status=123 blahblah"
# (i.e. the key + value on one line)
lines = data.split('\n')
if len(lines) == 1:
d = lines[0][len('stream-status='):]
# if there are actually 0 streams, then there's nothing
# left to parse
if len(d):
self._stream_update(d)
else:
[self._stream_update(line) for line in lines[1:]]
def _update_network_status(self, data):
"""
Used internally as a callback for updating Router information
from NS and NEWCONSENSUS events.
"""
for line in data.split('\n'):
self._network_status_parser.process(line)
txtorlog.msg(len(self.routers_by_name), "named routers found.")
## remove any names we added that turned out to have dups
for (k, v) in self.routers.items():
if v is None:
txtorlog.msg(len(self.routers_by_name[k]), "dups:", k)
del self.routers[k]
txtorlog.msg(len(self.guards), "GUARDs")
def _maybe_create_circuit(self, circ_id):
if circ_id not in self.circuits:
c = self.circuit_factory(self)
c.listen(self)
[c.listen(x) for x in self.circuit_listeners]
else:
c = self.circuits[circ_id]
return c
def _circuit_update(self, line):
"""
Used internally as a callback to update Circuit information
from CIRC events.
"""
#print "circuit_update",line
args = line.split()
circ_id = int(args[0])
c = self._maybe_create_circuit(circ_id)
c.update(args)
def _stream_update(self, line):
"""
Used internally as a callback to update Stream information
from STREAM events.
"""
#print "stream_update",line
if line.strip() == 'stream-status=':
## this happens if there are no active streams
return
args = line.split()
assert len(args) >= 3
stream_id = int(args[0])
wasnew = False
if stream_id not in self.streams:
stream = self.stream_factory(self)
self.streams[stream_id] = stream
stream.listen(self)
[stream.listen(x) for x in self.stream_listeners]
wasnew = True
self.streams[stream_id].update(args)
## if the update closed the stream, it won't be in our list
## anymore. FIXME: how can we ever hit such a case as the
## first update being a CLOSE?
if wasnew and stream_id in self.streams:
self._maybe_attach(self.streams[stream_id])
def _addr_map(self, addr):
"Internal callback to update DNS cache. Listens to ADDRMAP."
txtorlog.msg(" --> addr_map", addr)
self.addrmap.update(addr)
event_map = {
'STREAM': _stream_update,
'CIRC': _circuit_update,
'NS': _update_network_status,
'NEWCONSENSUS': _update_network_status,
'ADDRMAP': _addr_map
}
"""event_map used by add_events to map event_name -> unbound method"""
@defer.inlineCallbacks
def _add_events(self):
"""
Add listeners for all the events the controller is interested in.
"""
for (event, func) in self.event_map.items():
## the map contains unbound methods, so we bind them
## to self so they call the right thing
yield self.protocol.add_event_listener(
event, types.MethodType(func, self, TorState))
## ICircuitContainer
def find_circuit(self, circid):
"ICircuitContainer API"
return self.circuits[circid]
## IRouterContainer
def router_from_id(self, routerid):
"""IRouterContainer API"""
try:
return self.routers[routerid[:41]]
except KeyError:
if routerid[0] != '$':
raise # just re-raise the KeyError
router = Router(self.protocol)
idhash = routerid[1:41]
nick = ''
is_named = False
if len(routerid) > 41:
nick = routerid[42:]
is_named = routerid[41] == '='
router.update(nick, hashFromHexId(idhash), '0' * 27, 'unknown',
'unknown', '0', '0')
router.name_is_unique = is_named
self.routers[router.id_hex] = router
return router
## implement IStreamListener
def stream_new(self, stream):
"IStreamListener: a new stream has been created"
txtorlog.msg("stream_new", stream)
def stream_succeeded(self, stream):
"IStreamListener: stream has succeeded"
txtorlog.msg("stream_succeeded", stream)
def stream_attach(self, stream, circuit):
"""
IStreamListener: the stream has been attached to a circuit. It
seems you get an attach to None followed by an attach to real
circuit fairly frequently. Perhaps related to __LeaveStreamsUnattached?
"""
txtorlog.msg("stream_attach", stream.id, stream.target_host, " -> ",
circuit)
def stream_detach(self, stream, **kw):
"""
IStreamListener
"""
txtorlog.msg("stream_detach", stream.id)
def stream_closed(self, stream, **kw):
"""
IStreamListener: stream has been closed (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_closed", stream.id)
del self.streams[stream.id]
def stream_failed(self, stream, **kw):
"""
IStreamListener: stream failed for some reason (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_failed", stream.id)
del self.streams[stream.id]
## implement ICircuitListener
def circuit_launched(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_launched", circuit)
self.circuits[circuit.id] = circuit
def circuit_extend(self, circuit, router):
"ICircuitListener API"
txtorlog.msg("circuit_extend:", circuit.id, router)
def circuit_built(self, circuit):
"ICircuitListener API"
txtorlog.msg(
"circuit_built:", circuit.id,
"->".join("%s.%s" % (x.name, x.location.countrycode)
for x in circuit.path), circuit.streams)
def circuit_new(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_new:", circuit.id)
self.circuits[circuit.id] = circuit
def circuit_destroy(self, circuit):
"Used by circuit_closed and circuit_failed (below)"
txtorlog.msg("circuit_destroy:", circuit.id)
del self.circuits[circuit.id]
def circuit_closed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_closed", circuit)
self.circuit_destroy(circuit)
def circuit_failed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_failed", circuit, str(kw))
self.circuit_destroy(circuit)
class TorInfo(object):
"""
Implements some attribute magic over top of TorControlProtocol so
that all the available GETINFO values are gettable in a little
easier fashion. Dashes are replaced by underscores (since dashes
aren't valid in method/attribute names for Python). Some of the
magic methods will take a single string argument if the
corresponding Tor GETINFO would take one (in 'GETINFO info/names'
it will end with '/*', and the same in torspec). In either case,
the method returns a Deferred which will callback with the
requested value, always a string.
For example (see also examples/tor_info.py):
proto = TorControlProtocol()
#...
def cb(arg):
print arg
info = TorInfo(proto)
info.traffic.written().addCallback(cb)
info.ip_to_country('8.8.8.8').addCallback(cb)
For interactive use -- or even checking things progammatically -- TorInfo
pretends it only has attributes that coorespond to valid GETINFO calls.
So for example, dir(info) will only return all the currently valid top-level
things. In the above example this might be ['traffic', 'ip_to_country'] (of
course in practice this is a much longer list). And "dir(info.traffic)" might
return ['read', 'written']
For something similar to this for configuration (GETCONF, SETCONF) see
TorConfig which is quite a lot more complicated (internally) since you can set
config items.
NOTE that 'GETINFO config/*' is not supported as it's the only case that's not a
leaf, but theoretically a method.
"""
def __init__(self, control, errback=None):
self._setup = False
self.attrs = {}
'''After _setup is True, these are all we show as attributes.'''
self.protocol = ITorControlProtocol(control)
self.errback = errback
self.post_bootstrap = defer.Deferred()
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap)
else:
self.bootstrap()
def _add_attribute(self, n, v):
self.attrs[n] = v
## iterator protocol
def __getitem__(self, idx):
return object.__getattribute__(self, 'attrs').items()[idx][1]
def __len__(self):
return len(object.__getattribute__(self, 'attrs'))
## change our attribute behavior based on the value of _setup
def __getattribute__(self, name):
sup = super(TorInfo, self)
if sup.__getattribute__('_setup') is False:
return sup.__getattribute__(name)
attrs = sup.__getattribute__('attrs')
if name == '__members__':
return attrs.keys()
else:
try:
return attrs[name]
except KeyError:
if name == 'dump':
return object.__getattribute__(self, name)
raise AttributeError(name)
def bootstrap(self, *args):
d = self.protocol.get_info_raw("info/names").addCallback(self._do_setup)
if self.errback:
d.addErrback(self.errback)
d.addCallback(self._setup_complete)
return d
def dump(self):
for x in object.__getattribute__(self, 'attrs').values():
x.dump('')
def _do_setup(self, data):
# FIXME figure out why network-status doesn't work (get nothing back from
# Tor it seems, although stem does get an answer). this is a space-separated
# list of ~2500 OR id's; could it be that LineReceiver can't handle it?
added_magic = []
for line in data.split('\n'):
if line == "info/names=" or line == "OK" or line.strip() == '':
continue
#print "LINE:",line
(name, documentation) = line.split(' ', 1)
## FIXME think about this -- this is the only case where
## there's something that's a directory
## (i.e. MagicContainer) AND needs to be a ConfigMethod as
## well...but doesn't really seem very useful. Somewhat
## simpler to not support this case for now...
if name == 'config/*':
continue
if name.endswith('/*'):
## this takes an arg, so make a method
bits = name[:-2].split('/')
takes_arg = True
else:
bits = name.split('/')
takes_arg = False
mine = self
for bit in bits[:-1]:
bit = bit.replace('-', '_')
if bit in mine.attrs:
mine = mine.attrs[bit]
if not isinstance(mine, MagicContainer):
raise RuntimeError("Already had something: %s for %s" % (bit, name))
else:
c = MagicContainer(bit)
added_magic.append(c)
mine._add_attribute(bit, c)
mine = c
n = bits[-1].replace('-', '_')
if n in mine.attrs:
raise RuntimeError("Already had something: %s for %s" % (n, name))
mine._add_attribute(n, ConfigMethod('/'.join(bits), self.protocol, takes_arg))
for c in added_magic:
c._setup_complete()
return None
def _setup_complete(self, *args):
pb = self.post_bootstrap
self._setup = True
pb.callback(self)
class TorConfig(object):
"""
This class abstracts out Tor's config so that you don't have to
realize things like: in order to successfully set multiple listen
addresses, you must put them all (and the or-ports) in one SETCONF
call.
Also, it gives easy access to all the configuration options
present. This is done with lazy caching: the first time you access
a value, it asks the underlying Tor (via TorControlProtocol) and
thereafter caches the value; if you change it, a SETCONF is
issued.
When setting configuration values, they are cached locally and DO
NOT AFFECT the running Tor until you call save(). When getting
config items they will reflect the current state of Tor
(i.e. *not* what's been set since the last save())
Note that you do not need to call save() if you're just using
TorConfig to create a .torrc file or for input to launch_tor().
FIXME: It also listens on the CONF_CHANGED event to update the
cached data in the event other controllers (etc) changed it. (Only
exists in Git versions?)
FIXME: when is CONF_CHANGED introduced in Tor? Can we do anything
like it for prior versions?
FIXME:
- HiddenServiceOptions is special: GETCONF on it returns
several (well, two) values. Besides adding the two keys 'by
hand' do we need to do anything special? Can't we just depend
on users doing 'conf.hiddenservicedir = foo' AND
'conf.hiddenserviceport = bar' before a save() ?
- once I determine a value is default, is there any way to
actually get what this value is?
"""
def __init__(self, control=None):
if control is None:
self.protocol = None
self.__dict__['_slutty_'] = None
else:
self.protocol = ITorControlProtocol(control)
self.config = {}
'''Current configuration, by keys.'''
self.unsaved = {}
'''Configuration that has been changed since last save().'''
self.parsers = {}
'''Instances of the parser classes, subclasses of TorConfigType'''
self.post_bootstrap = defer.Deferred()
if self.protocol:
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap).addErrback(log.err)
else:
self.bootstrap()
else:
self.post_bootstrap.callback(self)
self.__dict__['_setup_'] = None
def __setattr__(self, name, value):
"""
we override this so that we can provide direct attribute
access to our config items, and move them into self.unsaved
when they've been changed. hiddenservices have to be special
unfortunately. the _setup_ thing is so that we can set up the
attributes we need in the constructor without uusing __dict__
all over the place.
"""
if '_setup_' in self.__dict__:
name = self._find_real_name(name)
if '_slutty_' not in self.__dict__ and name.lower() != 'hiddenservices':
value = self.parsers[name].validate(value, self, name)
if isinstance(value, types.ListType):
value = _ListWrapper(value, functools.partial(self.mark_unsaved, name))
name = self._find_real_name(name)
self.unsaved[name] = value
else:
super(TorConfig, self).__setattr__(name, value)
def __getattr__(self, name):
"""
on purpose, we don't return self.saved if the key is in there
because I want the config to represent the running Tor not
``things which might get into the running Tor if save() were
to be called''
"""
return self.config[self._find_real_name(name)]
def get_type(self, name):
"""
return the type of a config key.
:param: name the key
FIXME can we do something more-clever than this for client
code to determine what sort of thing a key is?
"""
if name.lower() == 'hiddenservices':
return HiddenService
return type(self.parsers[name])
def _conf_changed(self, arg):
"""
internal callback. from control-spec:
4.1.18. Configuration changed
The syntax is:
StartReplyLine *(MidReplyLine) EndReplyLine
StartReplyLine = "650-CONF_CHANGED" CRLF
MidReplyLine = "650-" KEYWORD ["=" VALUE] CRLF
EndReplyLine = "650 OK"
Tor configuration options have changed (such as via a SETCONF or RELOAD
signal). KEYWORD and VALUE specify the configuration option that was changed.
Undefined configuration options contain only the KEYWORD.
"""
conf = parse_keywords(arg, multiline_values=False)
for (k, v) in conf.items():
## v will be txtorcon.DEFAULT_VALUE already from
## parse_keywords if it was unspecified
self.config[self._find_real_name(k)] = v
def bootstrap(self, *args):
try:
self.protocol.add_event_listener('CONF_CHANGED', self._conf_changed)
except (RuntimeError, e):
## for Tor versions which don't understand CONF_CHANGED
## there's nothing we can really do.
log.msg("Can't listen for CONF_CHANGED event; won't stay up-to-date with other clients.")
return self.protocol.get_info_raw("config/names").addCallbacks(self._do_setup, log.err).addCallback(self.do_post_bootstrap).addErrback(log.err)
def do_post_bootstrap(self, *args):
self.post_bootstrap.callback(self)
self.__dict__['post_bootstrap'] = None
def needs_save(self):
return len(self.unsaved) > 0
def mark_unsaved(self, name):
name = self._find_real_name(name)
if name in self.config and name not in self.unsaved:
self.unsaved[name] = self.config[self._find_real_name(name)]
def save(self):
"""
Save any outstanding items. This returns a Deferred which will
errback if Tor was unhappy with anything, or callback with
this TorConfig object on success.
"""
if not self.needs_save():
return defer.succeed(self)
args = []
for (key, value) in self.unsaved.items():
if key == 'HiddenServices':
self.config['HiddenServices'] = value
for hs in value:
args.append('HiddenServiceDir')
args.append(hs.dir)
for p in hs.ports:
args.append('HiddenServicePort')
args.append(str(p))
if hs.version:
args.append('HiddenServiceVersion')
args.append(str(hs.version))
if hs.authorize_client:
args.append('HiddenServiceAuthorizeClient')
args.append(hs.authorize_client)
continue
if isinstance(value, types.ListType):
for x in value:
args.append(key)
args.append(str(x))
else:
args.append(key)
args.append(value)
# FIXME in future we should wait for CONF_CHANGED and
# update then, right?
self.config[self._find_real_name(key)] = value
## FIXME might want to re-think this, but currently there's no
## way to put things into a config and get them out again
## nicely...unless you just don't assign a protocol
if self.protocol:
d = self.protocol.set_conf(*args)
d.addCallback(self._save_completed)
d.addErrback(log.err)
return d
else:
self._save_completed()
return defer.succeed(self)
def _save_completed(self, *args):
'''internal callback'''
self.__dict__['unsaved'] = {}
return self
def _find_real_name(self, name):
for x in self.__dict__['config'].keys():
if x.lower() == name:
return x
return name
@defer.inlineCallbacks
def _do_setup(self, data):
for line in data.split('\n'):
if line == "config/names=" or line == "OK":
continue
(name, value) = line.split()
if name == 'HiddenServiceOptions':
## set up the "special-case" hidden service stuff
servicelines = yield self.protocol.get_conf_raw('HiddenServiceOptions')
self._setup_hidden_services(servicelines)
continue
if value == 'Dependant':
continue
## there's a thing called "Boolean+Auto" which is -1 for
## auto, 0 for false and 1 for true. could be nicer if it
## was called AutoBoolean or something, but...
value = value.replace('+', '_')
inst = None
# FIXME: put parser classes in dict instead?
for cls in config_types:
if cls.__name__ == value:
inst = cls()
if not inst:
raise RuntimeError("Don't have a parser for: " + value)
v = yield self.protocol.get_conf(name)
v = v[name]
self.parsers[name] = inst
if value == 'LineList':
## FIXME should move to the parse() method, but it
## doesn't have access to conf object etc.
self.config[self._find_real_name(name)] = _ListWrapper(self.parsers[name].parse(v), functools.partial(self.mark_unsaved, name))
else:
self.config[self._find_real_name(name)] = self.parsers[name].parse(v)
# can't just return in @inlineCallbacks-decorated methods
defer.returnValue(self)
def _setup_hidden_services(self, servicelines):
hs = []
directory = None
ports = []
ver = None
auth = None
for line in servicelines.split('\n'):
if not len(line.strip()):
continue
if line == 'HiddenServiceOptions':
continue
k, v = line.split('=')
if k == 'HiddenServiceDir':
if directory is not None:
hs.append(HiddenService(self, directory, ports, auth, ver))
directory = v
ports = []
ver = None
auth = None
elif k == 'HiddenServicePort':
ports.append(v)
elif k == 'HiddenServiceVersion':
ver = int(v)
elif k == 'HiddenServiceAuthorizeClient':
auth = v
else:
raise RuntimeError("Can't parse HiddenServiceOptions: " + k)
if directory is not None:
hs.append(HiddenService(self, directory, ports, auth, ver))
name = 'HiddenServices'
self.config[name] = _ListWrapper(hs, functools.partial(self.mark_unsaved, name))
def create_torrc(self):
rtn = StringIO()
for (k, v) in self.config.items() + self.unsaved.items():
if type(v) is _ListWrapper:
if k.lower() == 'hiddenservices':
for x in v:
for (kk, vv) in x.config_attributes():
rtn.write('%s %s\n' % (kk, vv))
else:
for x in v:
rtn.write('%s %s\n' % (k, x))
else:
rtn.write('%s %s\n' % (k, v))
return rtn.getvalue()
class TorState(object):
"""
This tracks the current state of Tor using a TorControlProtocol.
On setup it first queries the initial state of streams and
circuits. It then asks for updates via the listeners. It requires
an ITorControlProtocol instance. The control protocol doesn't need
to be bootstrapped yet. The Deferred .post_boostrap is driggered
when the TorState instance is fully ready to go. The easiest way
is to use the helper method
:func:`txtorcon.build_tor_connection`. For details, see the
implementation of that.
You may add an :class:`txtorcon.interface.IStreamAttacher` to
provide a custom mapping for Strams to Circuits (by default Tor
picks by itself).
This is also a good example of the various listeners, and acts as
an :class:`txtorcon.interface.ICircuitContainer` and
:class:`txtorcon.interface.IRouterContainer`.
:cvar DO_NOT_ATTACH:
Constant to return from an IAttacher indicating you don't want to
attach this stream at all.
"""
@classmethod
def from_protocol(cls, protocol, **kw):
'''
Create a new, boot-strapped TorState from a TorControlProtocol
instance.
:return: a Deferred that fires with a TorState instance
'''
state = TorState(protocol, bootstrap=True)
return state.post_bootstrap
def __init__(self, protocol, bootstrap=True):
self.protocol = ITorControlProtocol(protocol)
# fixme could use protocol.on_disconnect to re-connect; see issue #3
# could override these to get your own Circuit/Stream subclasses
# to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self._attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
#: keys on id (integer)
self.circuits = {}
#: keys on id (integer)
self.streams = {}
#: list of unique routers
self.all_routers = set()
#: keys by hexid (string) and by unique names
self.routers = {}
self._old_routers = {}
#: keys on name, value always list (many duplicate "Unnamed"
#: routers, for example)
self.routers_by_name = {}
#: keys by hexid (string)
self.routers_by_hash = {}
#: potentially-usable as entry guards, I think? (any router
#: with 'Guard' flag)
self.guards = {}
#: from GETINFO entry-guards, our current entry guards
self.entry_guards = {}
#: list of entry guards we didn't parse out
self.unusable_entry_guards = []
#: keys by name
self.authorities = {}
#: see set_attacher
self._cleanup = None
self._network_status_parser = MicrodescriptorParser(
self._create_router)
self.post_bootstrap = defer.Deferred()
if bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap)
self.protocol.post_bootstrap.addErrback(
self.post_bootstrap.errback)
def _create_router(self, **kw):
id_hex = hexIdFromHash(kw['idhash'])
try:
router = self._old_routers[id_hex]
except KeyError:
router = Router(self.protocol)
self.routers[id_hex] = router
router.from_consensus = True
router.update(
kw['nickname'],
kw['idhash'],
kw['orhash'],
kw['modified'],
kw['ip'],
kw['orport'],
kw['dirport'],
)
router.flags = kw.get('flags', [])
if 'bandwidth' in kw:
router.bandwidth = kw['bandwidth']
if 'ip_v6' in kw:
router.ip_v6.extend(kw['ip_v6'])
if 'guard' in router.flags:
self.guards[router.id_hex] = router
if 'authority' in router.flags:
self.authorities[router.name] = router
if router.name in self.routers:
self.routers[router.name] = None
else:
self.routers[router.name] = router
if router.name in self.routers_by_name:
self.routers_by_name[router.name].append(router)
else:
self.routers_by_name[router.name] = [router]
self.routers[router.id_hex] = router
self.routers_by_hash[router.id_hex] = router
self.all_routers.add(router)
@defer.inlineCallbacks
def _bootstrap(self, arg=None):
"This takes an arg so we can use it as a callback (see __init__)."
# update list of routers (must be before we do the
# circuit-status)
# look out! we're depending on get_info_incremental returning
# *lines*, which isn't documented -- but will be true because
# TorControlProtocol is a LineReceiver...
yield self.protocol.get_info_incremental(
'ns/all',
self._network_status_parser.feed_line,
)
self._network_status_parser.done()
# update list of existing circuits
cs = yield self.protocol.get_info_raw('circuit-status')
self._circuit_status(cs)
# update list of streams
ss = yield self.protocol.get_info_raw('stream-status')
self._stream_status(ss)
# update list of existing address-maps
key = 'address-mappings/all'
am = yield self.protocol.get_info_raw(key)
# strip addressmappsings/all= and OK\n from raw data
am = am[len(key) + 1:]
for line in am.split('\n'):
if len(line.strip()) == 0:
continue # FIXME
self.addrmap.update(line)
self._add_events()
entries = yield self.protocol.get_info_raw("entry-guards")
for line in entries.split('\n')[1:]:
if len(line.strip()) == 0 or line.strip() == 'OK':
# XXX does this ever really happen?
continue
args = line.split()
(name, status) = args[:2]
name = name[:41]
# this is sometimes redundant, as a missing entry guard
# usually means it won't be in our list of routers right
# now, but just being on the safe side
if status.lower() != 'up':
self.unusable_entry_guards.append(line)
continue
try:
self.entry_guards[name] = self.router_from_id(name)
except KeyError:
self.unusable_entry_guards.append(line)
# in case process/pid doesn't exist and we don't know the PID
# because we own it, we just leave it as 0 (previously
# guessed using psutil, but that only works if there's
# exactly one tor running anyway)
try:
pid = yield self.protocol.get_info_raw("process/pid")
except TorProtocolError:
pid = None
self.tor_pid = 0
if pid:
try:
pid = parse_keywords(pid)['process/pid']
self.tor_pid = int(pid)
except Exception: # fixme: ValueError and KeyError ..?
self.tor_pid = 0
if not self.tor_pid and self.protocol.is_owned:
self.tor_pid = self.protocol.is_owned
self.post_bootstrap.callback(self)
self.post_boostrap = None
def undo_attacher(self):
"""
Shouldn't Tor handle this by turning this back to 0 if the
controller that twiddled it disconnects?
"""
return self.protocol.set_conf("__LeaveStreamsUnattached", 0)
def set_attacher(self, attacher, myreactor):
"""
Provide an :class:`txtorcon.interface.IStreamAttacher` to
associate streams to circuits.
You are Strongly Encouraged to **not** use this API directly,
and instead use :meth:`txtorcon.Circuit.stream_via` or
:meth:`txtorcon.Circuit.web_agent` instead. If you do need to
use this API, it's an error if you call either of the other
two methods.
This won't get turned on until after bootstrapping is
completed. ('__LeaveStreamsUnattached' needs to be set to '1'
and the existing circuits list needs to be populated).
"""
react = IReactorCore(myreactor)
if attacher:
if self._attacher is attacher:
return
if self._attacher is not None:
raise RuntimeError(
"set_attacher called but we already have an attacher")
self._attacher = IStreamAttacher(attacher)
else:
self._attacher = None
if self._attacher is None:
d = self.undo_attacher()
if self._cleanup:
react.removeSystemEventTrigger(self._cleanup)
self._cleanup = None
else:
d = self.protocol.set_conf("__LeaveStreamsUnattached", "1")
self._cleanup = react.addSystemEventTrigger(
'before',
'shutdown',
self.undo_attacher,
)
return d
# noqa
stream_close_reasons = {
'REASON_MISC': 1, # (catch-all for unlisted reasons)
'REASON_RESOLVEFAILED': 2, # (couldn't look up hostname)
'REASON_CONNECTREFUSED': 3, # (remote host refused connection) [*]
'REASON_EXITPOLICY': 4, # (OR refuses to connect to host or port)
'REASON_DESTROY': 5, # (Circuit is being destroyed)
'REASON_DONE': 6, # (Anonymized TCP connection was closed)
'REASON_TIMEOUT':
7, # (Connection timed out, or OR timed out while connecting)
'REASON_NOROUTE':
8, # (Routing error while attempting to contact destination)
'REASON_HIBERNATING': 9, # (OR is temporarily hibernating)
'REASON_INTERNAL': 10, # (Internal error at the OR)
'REASON_RESOURCELIMIT': 11, # (OR has no resources to fulfill request)
'REASON_CONNRESET': 12, # (Connection was unexpectedly reset)
'REASON_TORPROTOCOL':
13, # (Sent when closing connection because of Tor protocol violations.)
'REASON_NOTDIRECTORY': 14
} # (Client sent RELAY_BEGIN_DIR to a non-directory relay.)
def close_stream(self, stream, reason='REASON_MISC', **kwargs):
"""
This sends a STREAMCLOSE command, using the specified reason
(either an int or one of the 14 strings in section 6.3 of
tor-spec.txt if the argument is a string). Any kwards are
passed through as flags if they evaluated to true
(e.g. "SomeFlag=True"). Currently there are none that Tor accepts.
"""
if type(stream) != int:
# assume it's a Stream instance
stream = stream.id
try:
reason = int(reason)
except ValueError:
try:
reason = TorState.stream_close_reasons[reason]
except KeyError:
raise ValueError('Unknown stream close reason "%s"' %
str(reason))
flags = flags_from_dict(kwargs)
# stream is now an ID no matter what we passed in
cmd = 'CLOSESTREAM %d %d%s' % (stream, reason, flags)
return self.protocol.queue_command(cmd)
def close_circuit(self, circid, **kwargs):
"""
This sends a CLOSECIRCUIT command, using any keyword arguments
passed as the Flags (currently, that is just 'IfUnused' which
means to only close the circuit when it is no longer used by
any streams).
:param circid:
Either a circuit-id (int) or a Circuit instance
:return:
a Deferred which callbacks with the result of queuing the
command to Tor (usually "OK"). If you want to instead know
when the circuit is actually-gone, see
:meth:`Circuit.close <txtorcon.circuit.Circuit.close>`
"""
if type(circid) != int:
# assume it's a Circuit instance
circid = circid.id
flags = flags_from_dict(kwargs)
return self.protocol.queue_command('CLOSECIRCUIT %s%s' %
(circid, flags))
def add_circuit_listener(self, icircuitlistener):
listen = ICircuitListener(icircuitlistener)
for circ in self.circuits.values():
circ.listen(listen)
self.circuit_listeners.append(listen)
def add_stream_listener(self, istreamlistener):
listen = IStreamListener(istreamlistener)
for stream in self.streams.values():
stream.listen(listen)
self.stream_listeners.append(listen)
def _find_circuit_after_extend(self, x):
ex, circ_id = x.split()
if ex != 'EXTENDED':
raise RuntimeError('Expected EXTENDED, got "%s"' % x)
circ_id = int(circ_id)
circ = self._maybe_create_circuit(circ_id)
circ.update([str(circ_id), 'EXTENDED'])
return circ
def build_circuit(self, routers=None, using_guards=True):
"""
Builds a circuit consisting of exactly the routers specified,
in order. This issues an EXTENDCIRCUIT call to Tor with all
the routers specified.
:param routers: a list of Router instances which is the path
desired. To allow Tor to choose the routers itself, pass
None (the default) for routers.
:param using_guards: A warning is issued if the first router
isn't in self.entry_guards.
:return:
A Deferred that will callback with a Circuit instance
(with the .id member being valid, and probably nothing
else).
"""
if routers is None or routers == []:
cmd = "EXTENDCIRCUIT 0"
else:
if using_guards and routers[0] not in self.entry_guards.values():
warnings.warn(
"Circuit doesn't start with a guard: %s" % routers,
RuntimeWarning)
cmd = "EXTENDCIRCUIT 0 "
first = True
for router in routers:
if first:
first = False
else:
cmd += ','
# XXX should we really accept bytes here?
if isinstance(router, bytes) and len(router) == 40 \
and hashFromHexId(router):
cmd += router.decode('utf8')
else:
cmd += router.id_hex[1:]
d = self.protocol.queue_command(cmd)
d.addCallback(self._find_circuit_after_extend)
return d
DO_NOT_ATTACH = object()
def _maybe_attach(self, stream):
"""
If we've got a custom stream-attachment instance (see
set_attacher) this will ask it for the appropriate
circuit. Note that we ignore .exit URIs and let Tor deal with
those (by passing circuit ID 0).
The stream attacher is allowed to return a Deferred which will
callback with the desired circuit.
You may return the special object DO_NOT_ATTACH which will
cause the circuit attacher to simply ignore the stream
(neither attaching it, nor telling Tor to attach it).
"""
if self._attacher is None:
return None
if stream.target_host is not None \
and '.exit' in stream.target_host:
# we want to totally ignore .exit URIs as these are
# used to specify a particular exit node, and trying
# to do STREAMATTACH on them will fail with an error
# from Tor anyway.
txtorlog.msg("ignore attacher:", stream)
return
# handle async or sync .attach() the same
circ_d = defer.maybeDeferred(
self._attacher.attach_stream,
stream,
self.circuits,
)
circ_d.addCallback(maybe_coroutine)
# actually do the attachment logic; .attach() can return 3 things:
# 1. None: let Tor do whatever it wants
# 2. DO_NOT_ATTACH: don't attach the stream at all
# 3. Circuit instance: attach to the provided circuit
def issue_stream_attach(circ):
txtorlog.msg("circuit:", circ)
if circ is None or circ is TorState.DO_NOT_ATTACH:
# tell Tor to do what it likes
return self.protocol.queue_command(b"ATTACHSTREAM %d 0" %
stream.id)
else:
# should get a Circuit instance; check it for suitability
if not isinstance(circ, Circuit):
raise RuntimeError(
"IStreamAttacher.attach() must return a Circuit instance "
"(or None or DO_NOT_ATTACH): %s")
if circ.id not in self.circuits:
raise RuntimeError(
"Attacher returned a circuit unknown to me.")
if circ.state != 'BUILT':
raise RuntimeError(
"Can only attach to BUILT circuits; %d is in %s." %
(circ.id, circ.state))
# we've got a valid Circuit instance; issue the command
return self.protocol.queue_command(b"ATTACHSTREAM %d %d" %
(stream.id, circ.id))
circ_d.addCallback(issue_stream_attach)
circ_d.addErrback(self._attacher_error)
return circ_d
def _attacher_error(self, fail):
"""
not ideal, but there's not really a good way to let the caller
handler errors :/ since we ultimately call this due to an
async request from Tor. Mostly these errors will be logic or
syntax errors in the caller's code anyway.
tests monkey-patch this to reduce spew
"""
print("Failure while attaching stream:", fail)
return fail
def _circuit_status(self, data):
"""Used internally as a callback for updating Circuit information"""
data = data[len('circuit-status='):].split('\n')
# sometimes there's a newline after circuit-status= and
# sometimes not, so we get rid of it
if len(data) and len(data[0].strip()) == 0:
data = data[1:]
for line in data:
self._circuit_update(line)
def _stream_status(self, data):
"Used internally as a callback for updating Stream information"
# there's a slight issue with a single-stream vs >= 2 streams,
# in that in the latter case we have a line by itself with
# "stream-status=" on it followed by the streams EXCEPT in the
# single-stream case which has "stream-status=123 blahblah"
# (i.e. the key + value on one line)
lines = data.split('\n')
if len(lines) == 1:
d = lines[0][len('stream-status='):]
# if there are actually 0 streams, then there's nothing
# left to parse
if len(d):
self._stream_update(d)
else:
[self._stream_update(line) for line in lines[1:]]
def _update_network_status(self, data):
"""
Used internally as a callback for updating Router information
from NEWCONSENSUS events.
"""
# XXX why are we ever getting this with 0 data?
if len(data):
self._old_routers = self.routers
self.routers = dict()
self.all_routers = set()
self.routers_by_hash = dict()
self.routers_by_name = dict()
for line in data.split('\n'):
self._network_status_parser.feed_line(line)
self._network_status_parser.done()
txtorlog.msg(len(self.routers_by_name), "named routers found.")
# remove any names we added that turned out to have dups
remove_keys = set()
for (k, v) in self.routers.items():
if v is None:
txtorlog.msg(len(self.routers_by_name[k]), "dups:", k)
remove_keys.add(k)
for k in remove_keys:
del self.routers[k]
txtorlog.msg(len(self.guards), "GUARDs")
def _maybe_create_circuit(self, circ_id):
if circ_id not in self.circuits:
c = self.circuit_factory(self)
c.listen(self)
for listener in self.circuit_listeners:
c.listen(listener)
else:
c = self.circuits[circ_id]
return c
def _circuit_update(self, line):
"""
Used internally as a callback to update Circuit information
from CIRC events.
"""
# print("circuit_update", line)
args = line.split()
circ_id = int(args[0])
c = self._maybe_create_circuit(circ_id)
c.update(args)
def _stream_update(self, line):
"""
Used internally as a callback to update Stream information
from STREAM events.
"""
if line.strip() == 'stream-status=':
# this happens if there are no active streams
return
args = line.split()
assert len(args) >= 3
stream_id = int(args[0])
wasnew = False
if stream_id not in self.streams:
stream = self.stream_factory(self, self.addrmap)
self.streams[stream_id] = stream
stream.listen(self)
for x in self.stream_listeners:
stream.listen(x)
wasnew = True
self.streams[stream_id].update(args)
# if the update closed the stream, it won't be in our list
# anymore. FIXME: how can we ever hit such a case as the
# first update being a CLOSE?
if wasnew and stream_id in self.streams:
self._maybe_attach(self.streams[stream_id])
def _addr_map(self, addr):
"Internal callback to update DNS cache. Listens to ADDRMAP."
txtorlog.msg(" --> addr_map", addr)
self.addrmap.update(addr)
event_map = {
'STREAM': _stream_update,
'CIRC': _circuit_update,
'NEWCONSENSUS': _update_network_status,
'ADDRMAP': _addr_map
}
"""event_map used by add_events to map event_name -> unbound method"""
@defer.inlineCallbacks
def _add_events(self):
"""
Add listeners for all the events the controller is interested in.
"""
for (event, func) in self.event_map.items():
# the map contains unbound methods, so we bind them
# to self so they call the right thing
try:
bound = types.MethodType(func, self, TorState)
except TypeError:
# python3
bound = types.MethodType(func, self)
yield self.protocol.add_event_listener(
event,
bound,
)
# ICircuitContainer
def find_circuit(self, circid):
"ICircuitContainer API"
return self.circuits[circid]
# IRouterContainer
def router_from_id(self, routerid):
"""IRouterContainer API"""
try:
return self.routers[routerid[:41]]
except KeyError:
if routerid[0] != '$':
raise # just re-raise the KeyError
router = Router(self.protocol)
idhash = routerid[1:41]
nick = ''
is_named = False
if len(routerid) > 41:
nick = routerid[42:]
is_named = routerid[41] == '='
router.update(nick, hashFromHexId(idhash), '0' * 27, 'unknown',
'unknown', '0', '0')
router.name_is_unique = is_named
self.routers[router.id_hex] = router
return router
# implement IStreamListener
def stream_new(self, stream):
"IStreamListener: a new stream has been created"
txtorlog.msg("stream_new", stream)
def stream_succeeded(self, stream):
"IStreamListener: stream has succeeded"
txtorlog.msg("stream_succeeded", stream)
def stream_attach(self, stream, circuit):
"""
IStreamListener: the stream has been attached to a circuit. It
seems you get an attach to None followed by an attach to real
circuit fairly frequently. Perhaps related to __LeaveStreamsUnattached?
"""
txtorlog.msg("stream_attach", stream.id, stream.target_host, " -> ",
circuit)
def stream_detach(self, stream, **kw):
"""
IStreamListener
"""
txtorlog.msg("stream_detach", stream.id)
def stream_closed(self, stream, **kw):
"""
IStreamListener: stream has been closed (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_closed", stream.id)
del self.streams[stream.id]
def stream_failed(self, stream, **kw):
"""
IStreamListener: stream failed for some reason (won't be in
controller's list anymore)
"""
txtorlog.msg("stream_failed", stream.id)
del self.streams[stream.id]
# implement ICircuitListener
def circuit_launched(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_launched", circuit)
self.circuits[circuit.id] = circuit
def circuit_extend(self, circuit, router):
"ICircuitListener API"
txtorlog.msg("circuit_extend:", circuit.id, router)
def circuit_built(self, circuit):
"ICircuitListener API"
txtorlog.msg(
"circuit_built:", circuit.id,
"->".join("%s.%s" % (x.name, x.location.countrycode)
for x in circuit.path), circuit.streams)
def circuit_new(self, circuit):
"ICircuitListener API"
txtorlog.msg("circuit_new:", circuit.id)
self.circuits[circuit.id] = circuit
def circuit_destroy(self, circuit):
"Used by circuit_closed and circuit_failed (below)"
txtorlog.msg("circuit_destroy:", circuit.id)
circuit._when_built.fire(
Failure(Exception("Destroying circuit; will never hit BUILT")))
del self.circuits[circuit.id]
def circuit_closed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_closed", circuit)
circuit._when_built.fire(
Failure(
Exception("Circuit closed ('{}')".format(
_extract_reason(kw)))))
self.circuit_destroy(circuit)
def circuit_failed(self, circuit, **kw):
"ICircuitListener API"
txtorlog.msg("circuit_failed", circuit, str(kw))
circuit._when_built.fire(
Failure(
Exception("Circuit failed ('{}')".format(
_extract_reason(kw)))))
self.circuit_destroy(circuit)
class TorInfo(object):
"""
Implements some attribute magic over top of TorControlProtocol so
that all the available GETINFO values are gettable in a little
easier fashion. Dashes are replaced by underscores (since dashes
aren't valid in method/attribute names for Python). Some of the
magic methods will take a single string argument if the
corresponding Tor GETINFO would take one (in 'GETINFO info/names'
it will end with '/*', and the same in torspec). In either case,
the method returns a Deferred which will callback with the
requested value, always a string.
For example (see also examples/tor_info.py):
proto = TorControlProtocol()
#...
def cb(arg):
print arg
info = TorInfo(proto)
info.traffic.written().addCallback(cb)
info.ip_to_country('8.8.8.8').addCallback(cb)
For interactive use -- or even checking things progammatically -- TorInfo
pretends it only has attributes that coorespond to valid GETINFO calls.
So for example, dir(info) will only return all the currently valid top-level
things. In the above example this might be ['traffic', 'ip_to_country'] (of
course in practice this is a much longer list). And "dir(info.traffic)" might
return ['read', 'written']
For something similar to this for configuration (GETCONF, SETCONF) see
TorConfig which is quite a lot more complicated (internally) since you can set
config items.
NOTE that 'GETINFO config/*' is not supported as it's the only case that's not a
leaf, but theoretically a method.
"""
def __init__(self, control, errback=None):
self._setup = False
self.attrs = {}
'''After _setup is True, these are all we show as attributes.'''
self.protocol = ITorControlProtocol(control)
self.errback = errback
self.post_bootstrap = defer.Deferred()
if self.protocol.post_bootstrap:
self.protocol.post_bootstrap.addCallback(self.bootstrap)
else:
self.bootstrap()
def _add_attribute(self, n, v):
self.attrs[n] = v
## iterator protocol
def __getitem__(self, idx):
return object.__getattribute__(self, 'attrs').items()[idx][1]
def __len__(self):
return len(object.__getattribute__(self, 'attrs'))
## change our attribute behavior based on the value of _setup
def __getattribute__(self, name):
sup = super(TorInfo, self)
if sup.__getattribute__('_setup') == False:
return sup.__getattribute__(name)
attrs = sup.__getattribute__('attrs')
if name == '__members__':
return attrs.keys()
else:
try:
return attrs[name]
except KeyError:
if name == 'dump':
return object.__getattribute__(self, name)
raise AttributeError(name)
def bootstrap(self, *args):
d = self.protocol.get_info_raw("info/names").addCallback(self._do_setup)
if self.errback:
d.addErrback(self.errback)
d.addCallback(self._setup_complete)
return d
def dump(self):
for x in object.__getattribute__(self, 'attrs').values():
x.dump('')
def _do_setup(self, data):
# FIXME figure out why network-status doesn't work (get nothing back from
# Tor it seems, although stem does get an answer). this is a space-separated
# list of ~2500 OR id's; could it be that LineReceiver can't handle it?
added_magic = []
for line in data.split('\n'):
if line == "info/names=" or line == "OK" or line.strip() == '':
continue
#print "LINE:",line
(name, documentation) = line.split(' ', 1)
## FIXME think about this -- this is the only case where
## there's something that's a directory
## (i.e. MagicContainer) AND needs to be a ConfigMethod as
## well...but doesn't really seem very useful. Somewhat
## simpler to not support this case for now...
if name == 'config/*':
continue
if name.endswith('/*'):
## this takes an arg, so make a method
bits = name[:-2].split('/')
takes_arg = True
else:
bits = name.split('/')
takes_arg = False
mine = self
for bit in bits[:-1]:
bit = bit.replace('-', '_')
if mine.attrs.has_key(bit):
mine = mine.attrs[bit]
if not isinstance(mine, MagicContainer):
raise RuntimeError("Already had something: %s for %s" % (bit, name))
else:
c = MagicContainer(bit)
added_magic.append(c)
mine._add_attribute(bit, c)
mine = c
n = bits[-1].replace('-', '_')
if mine.attrs.has_key(n):
raise RuntimeError("Already had something: %s for %s" % (n, name))
mine._add_attribute(n, ConfigMethod('/'.join(bits), self.protocol, takes_arg))
for c in added_magic:
c._setup_complete()
return None
def _setup_complete(self, *args):
pb = self.post_bootstrap
self._setup = True
pb.callback(self)
def __init__(self, protocol, bootstrap=True):
self.protocol = ITorControlProtocol(protocol)
# fixme could use protocol.on_disconnect to re-connect; see issue #3
# could override these to get your own Circuit/Stream subclasses
# to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self._attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = 'tor'
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
#: keys on id (integer)
self.circuits = {}
#: keys on id (integer)
self.streams = {}
#: list of unique routers
self.all_routers = set()
#: keys by hexid (string) and by unique names
self.routers = {}
self._old_routers = {}
#: keys on name, value always list (many duplicate "Unnamed"
#: routers, for example)
self.routers_by_name = {}
#: keys by hexid (string)
self.routers_by_hash = {}
#: potentially-usable as entry guards, I think? (any router
#: with 'Guard' flag)
self.guards = {}
#: from GETINFO entry-guards, our current entry guards
self.entry_guards = {}
#: list of entry guards we didn't parse out
self.unusable_entry_guards = []
#: keys by name
self.authorities = {}
#: see set_attacher
self._cleanup = None
self._network_status_parser = MicrodescriptorParser(
self._create_router)
self.post_bootstrap = defer.Deferred()
if bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap)
self.protocol.post_bootstrap.addErrback(
self.post_bootstrap.errback)
def __init__(self, protocol, bootstrap=True):
self.protocol = ITorControlProtocol(protocol)
# fixme could use protocol.on_disconnect to re-connect; see issue #3
# could override these to get your own Circuit/Stream subclasses
# to track these things
self.circuit_factory = Circuit
self.stream_factory = Stream
self.attacher = None
"""If set, provides
:class:`txtorcon.interface.IStreamAttacher` to attach new
streams we hear about."""
self.tor_binary = "tor"
self.circuit_listeners = []
self.stream_listeners = []
self.addrmap = AddrMap()
self.circuits = {} # keys on id (integer)
self.streams = {} # keys on id (integer)
self.all_routers = set() # list of unique routers
self.routers = {} # keys by hexid (string) and by unique names
self.routers_by_name = {} # keys on name, value always list (many duplicate "Unnamed" routers, for example)
self.routers_by_hash = {} # keys by hexid (string)
self.guards = {} # potentially-usable as entry guards, I think? (any router with 'Guard' flag)
self.entry_guards = {} # from GETINFO entry-guards, our current entry guards
self.unusable_entry_guards = [] # list of entry guards we didn't parse out
self.authorities = {} # keys by name
self.cleanup = None # see set_attacher
class die(object):
__name__ = "die" # FIXME? just to ease spagetti.py:82's pain
def __init__(self, msg):
self.msg = msg
def __call__(self, *args):
raise RuntimeError(self.msg % tuple(args))
waiting_r = State("waiting_r")
waiting_w = State("waiting_w")
waiting_p = State("waiting_p")
waiting_s = State("waiting_s")
def ignorable_line(x):
x = x.strip()
return x in [".", "OK", ""] or x.startswith("ns/")
waiting_r.add_transition(Transition(waiting_r, ignorable_line, None))
waiting_r.add_transition(Transition(waiting_s, lambda x: x.startswith("r "), self._router_begin))
# FIXME use better method/func than die!!
waiting_r.add_transition(
Transition(waiting_r, lambda x: not x.startswith("r "), die('Expected "r " while parsing routers not "%s"'))
)
waiting_s.add_transition(Transition(waiting_w, lambda x: x.startswith("s "), self._router_flags))
waiting_s.add_transition(Transition(waiting_s, lambda x: x.startswith("a "), self._router_address))
waiting_s.add_transition(Transition(waiting_r, ignorable_line, None))
waiting_s.add_transition(
Transition(
waiting_r,
lambda x: not x.startswith("s ") and not x.startswith("a "),
die('Expected "s " while parsing routers not "%s"'),
)
)
waiting_s.add_transition(Transition(waiting_r, lambda x: x.strip() == ".", None))
waiting_w.add_transition(Transition(waiting_p, lambda x: x.startswith("w "), self._router_bandwidth))
waiting_w.add_transition(Transition(waiting_r, ignorable_line, None))
waiting_w.add_transition(
Transition(waiting_s, lambda x: x.startswith("r "), self._router_begin)
) # "w" lines are optional
waiting_w.add_transition(
Transition(waiting_r, lambda x: not x.startswith("w "), die('Expected "w " while parsing routers not "%s"'))
)
waiting_w.add_transition(Transition(waiting_r, lambda x: x.strip() == ".", None))
waiting_p.add_transition(Transition(waiting_r, lambda x: x.startswith("p "), self._router_policy))
waiting_p.add_transition(Transition(waiting_r, ignorable_line, None))
waiting_p.add_transition(
Transition(waiting_s, lambda x: x.startswith("r "), self._router_begin)
) # "p" lines are optional
waiting_p.add_transition(
Transition(waiting_r, lambda x: x[:2] != "p ", die('Expected "p " while parsing routers not "%s"'))
)
waiting_p.add_transition(Transition(waiting_r, lambda x: x.strip() == ".", None))
self._network_status_parser = FSM([waiting_r, waiting_s, waiting_w, waiting_p])
self.post_bootstrap = defer.Deferred()
if bootstrap:
self.protocol.post_bootstrap.addCallback(self._bootstrap)
self.protocol.post_bootstrap.addErrback(self.post_bootstrap.errback)
