def runCoroutine(reactor: IReactorTime,
routine: Coroutine[None, None, None]
) -> None:
"""Run the given coroutine, while returning control to the reactor
after every processing step.
Use `await asyncio.sleep(0)` in your coroutine to separate processing
steps.
"""
try:
routine.send(None)
except StopIteration:
pass
else:
reactor.callLater(0, runCoroutine, reactor, routine)
class TelemetryStore(CollectionState):
"""
Accepts telemetry messages and exports the accumulated information obtained from them.
"""
implements(ITelemetryStore)
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
self.__flush_call = None
# not exported
def receive(self, message):
"""Store the supplied telemetry message object."""
message = ITelemetryMessage(message)
object_id = unicode(message.get_object_id())
if object_id in self.__objects:
obj = self.__objects[object_id]
else:
obj = self.__objects[object_id] = ITelemetryObject(
# TODO: Should probably have a context object supplying last message time and delete_me()
message.get_object_constructor()(object_id=object_id))
obj.receive(message)
expiry = obj.get_object_expiry()
self.__expiry_times[object_id] = expiry
if obj.is_interesting():
self.__interesting_objects[object_id] = obj
self.__maybe_schedule_flush()
def __flush_expired(self):
current_time = self.__time_source.seconds()
deletes = []
for object_id, expiry in self.__expiry_times.iteritems():
if expiry <= current_time:
deletes.append(object_id)
for object_id in deletes:
del self.__objects[object_id]
del self.__expiry_times[object_id]
if object_id in self.__interesting_objects:
del self.__interesting_objects[object_id]
self.__maybe_schedule_flush()
def __maybe_schedule_flush(self):
"""Schedule a call to __flush_expired if there is not one already."""
if self.__flush_call and self.__flush_call.active():
# Could need to schedule one earlier than already scheduled.
self.__flush_call.cancel()
if self.__expiry_times:
next_expiry = min(self.__expiry_times.itervalues())
self.__flush_call = self.__time_source.callLater(
next_expiry - self.__time_source.seconds(),
self.__flush_expired)
class BandwidthUpdater:
def __init__(self, config, scheduler):
self.bandwidth = 0
self.config = config
self.scheduler = IReactorTime(scheduler)
self.generator = self.next_update()
def next_update(self):
"""
Generator that gives out the next time to do a bandwidth update,
as well as what the new bandwidth value should be. Here, we toggle
the bandwidth every 20 minutes.
"""
while True:
if self.bandwidth:
self.bandwidth = 0
self.burst = 0
else:
self.bandwidth = 20 * 1024 * 1024
self.burst = self.bandwidth
yield (datetime.datetime.now() + datetime.timedelta(minutes=20),
self.bandwidth, self.burst)
def do_update(self):
x = self.generator.next()
future = x[0]
self.new_bandwidth = x[1]
self.new_burst = x[2]
tm = (future - datetime.datetime.now()).seconds
self.scheduler.callLater(tm, self.really_update)
print "waiting", tm, "seconds to adjust bandwidth"
def really_update(self):
print "setting bandwidth + burst to", self.new_bandwidth, self.new_burst
self.config.set_config('BandWidthBurst', self.new_burst,
'BandWidthRate', self.new_bandwidth)
self.doUpdate()
class BandwidthUpdater:
def __init__(self, config, scheduler):
self.bandwidth = 0
self.config = config
self.scheduler = IReactorTime(scheduler)
self.generator = self.next_update()
def next_update(self):
"""
Generator that gives out the next time to do a bandwidth update,
as well as what the new bandwidth value should be. Here, we toggle
the bandwidth every 20 minutes.
"""
while True:
if self.bandwidth:
self.bandwidth = 0
self.burst = 0
else:
self.bandwidth = 20 * 1024 * 1024
self.burst = self.bandwidth
yield (datetime.datetime.now() + datetime.timedelta(minutes=20),
self.bandwidth, self.burst)
def do_update(self):
x = self.generator.next()
future = x[0]
self.new_bandwidth = x[1]
self.new_burst = x[2]
tm = (future - datetime.datetime.now()).seconds
self.scheduler.callLater(tm, self.really_update)
print "waiting", tm, "seconds to adjust bandwidth"
def really_update(self):
print "setting bandwidth + burst to", self.new_bandwidth, self.new_burst
self.config.set_config('BandWidthBurst', self.new_burst,
'BandWidthRate', self.new_bandwidth)
self.doUpdate()
class EventualQueue(object):
def __init__(self, clock):
# pass clock=reactor unless you're testing
self._clock = IReactorTime(clock)
self._calls = []
self._flush_d = None
self._timer = None
def eventually(self, f, *args, **kwargs):
self._calls.append((f, args, kwargs))
if not self._timer:
self._timer = self._clock.callLater(0, self._turn)
def fire_eventually(self, value=None):
d = Deferred()
self.eventually(d.callback, value)
return d
def _turn(self):
while self._calls:
(f, args, kwargs) = self._calls.pop(0)
try:
f(*args, **kwargs)
except Exception:
log.err()
self._timer = None
d, self._flush_d = self._flush_d, None
if d:
d.callback(None)
def flush_sync(self):
# if you have control over the Clock, this will synchronously flush the
# queue
assert self._clock.advance, "needs clock=twisted.internet.task.Clock()"
while self._calls:
self._clock.advance(0)
def flush(self):
# this is for unit tests, not application code
assert not self._flush_d, "only one flush at a time"
self._flush_d = Deferred()
self.eventually(lambda: None)
return self._flush_d
def deferLater(
clock: IReactorTime,
delay: float,
callable: Optional[Callable[..., _T]] = None,
*args: object,
**kw: object,
) -> Deferred[_T]:
"""
Call the given function after a certain period of time has passed.
@param clock: The object which will be used to schedule the delayed
call.
@param delay: The number of seconds to wait before calling the function.
@param callable: The callable to call after the delay, or C{None}.
@param args: The positional arguments to pass to C{callable}.
@param kw: The keyword arguments to pass to C{callable}.
@return: A deferred that fires with the result of the callable when the
specified time has elapsed.
"""
def deferLaterCancel(deferred: Deferred[object]) -> None:
delayedCall.cancel()
def cb(result: object) -> _T:
if callable is None:
return None # type: ignore[return-value]
return callable(*args, **kw)
d: Deferred[_T] = Deferred(deferLaterCancel)
d.addCallback(cb)
delayedCall = clock.callLater(delay, d.callback, None)
return d
class TelemetryStore(CollectionState):
"""
Accepts telemetry messages and exports the accumulated information obtained from them.
"""
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
self.__flush_call = None
# not exported
def receive(self, message):
"""Store the supplied telemetry message object."""
message = ITelemetryMessage(message)
object_id = unicode(message.get_object_id())
if object_id in self.__objects:
obj = self.__objects[object_id]
else:
obj = self.__objects[object_id] = ITelemetryObject(
# TODO: Should probably have a context object supplying last message time and delete_me()
message.get_object_constructor()(object_id=object_id))
obj.receive(message)
expiry = obj.get_object_expiry()
self.__expiry_times[object_id] = expiry
if obj.is_interesting():
self.__interesting_objects[object_id] = obj
self.__maybe_schedule_flush()
def __flush_expired(self):
current_time = self.__time_source.seconds()
deletes = []
for object_id, expiry in self.__expiry_times.iteritems():
if expiry <= current_time:
deletes.append(object_id)
for object_id in deletes:
del self.__objects[object_id]
del self.__expiry_times[object_id]
if object_id in self.__interesting_objects:
del self.__interesting_objects[object_id]
self.__maybe_schedule_flush()
def __maybe_schedule_flush(self):
"""Schedule a call to __flush_expired if there is not one already."""
if self.__flush_call and self.__flush_call.active():
# Could need to schedule one earlier than already scheduled.
self.__flush_call.cancel()
if self.__expiry_times:
now = self.__time_source.seconds()
next_expiry = min(self.__expiry_times.itervalues())
sec_until_expiry = max(0, next_expiry - now)
self.__flush_call = self.__time_source.callLater(
sec_until_expiry,
self.__flush_expired)
def __init__(self,
connection_creator,
progress_updates=None,
config=None,
ireactortime=None,
timeout=None,
kill_on_stderr=True,
stdout=None,
stderr=None):
"""
This will read the output from a Tor process and attempt a
connection to its control port when it sees any 'Bootstrapped'
message on stdout. You probably don't need to use this
directly except as the return value from the
:func:`txtorcon.launch_tor` method. tor_protocol contains a
valid :class:`txtorcon.TorControlProtocol` instance by that
point.
connection_creator is a callable that should return a Deferred
that callbacks with a :class:`txtorcon.TorControlProtocol`;
see :func:`txtorcon.launch_tor` for the default one which is a
functools.partial that will call
``connect(TorProtocolFactory())`` on an appropriate
:api:`twisted.internet.endpoints.TCP4ClientEndpoint`
:param connection_creator: A no-parameter callable which
returns a Deferred which promises a
:api:`twisted.internet.interfaces.IStreamClientEndpoint
<IStreamClientEndpoint>`. If this is None, we do NOT
attempt to connect to the underlying Tor process.
:param progress_updates: A callback which received progress
updates with three args: percent, tag, summary
:param config: a TorConfig object to connect to the
TorControlProtocl from the launched tor (should it succeed)
:param ireactortime:
An object implementing IReactorTime (i.e. a reactor) which
needs to be supplied if you pass a timeout.
:param timeout:
An int representing the timeout in seconds. If we are
unable to reach 100% by this time we will consider the
setting up of Tor to have failed. Must supply ireactortime
if you supply this.
:param kill_on_stderr:
When True, kill subprocess if we receive anything on stderr
:param stdout:
Anything subprocess writes to stdout is sent to .write() on this
:param stderr:
Anything subprocess writes to stderr is sent to .write() on this
:ivar tor_protocol: The TorControlProtocol instance connected
to the Tor this
:api:`twisted.internet.protocol.ProcessProtocol
<ProcessProtocol>`` is speaking to. Will be valid after
the Deferred returned from
:meth:`TorProcessProtocol.when_connected` is triggered.
"""
self.config = config
self.tor_protocol = None
self.progress_updates = progress_updates
# XXX if connection_creator is not None .. is connected_cb
# tied to connection_creator...?
if connection_creator:
self.connection_creator = connection_creator
else:
self.connection_creator = None
# use SingleObserver
self._connected_listeners = [
] # list of Deferred (None when we're connected)
self.attempted_connect = False
self.to_delete = []
self.kill_on_stderr = kill_on_stderr
self.stderr = stderr
self.stdout = stdout
self.collected_stdout = StringIO()
self._setup_complete = False
self._did_timeout = False
self._timeout_delayed_call = None
self._on_exit = [] # Deferred's we owe a call/errback to when we exit
if timeout:
if not ireactortime:
raise RuntimeError(
'Must supply an IReactorTime object when supplying a '
'timeout')
ireactortime = IReactorTime(ireactortime)
self._timeout_delayed_call = ireactortime.callLater(
timeout, self._timeout_expired)
def __init__(self, connection_creator, progress_updates=None, config=None,
ireactortime=None, timeout=None, kill_on_stderr=True,
stdout=None, stderr=None):
"""
This will read the output from a Tor process and attempt a
connection to its control port when it sees any 'Bootstrapped'
message on stdout. You probably don't need to use this
directly except as the return value from the
:func:`txtorcon.launch_tor` method. tor_protocol contains a
valid :class:`txtorcon.TorControlProtocol` instance by that
point.
connection_creator is a callable that should return a Deferred
that callbacks with a :class:`txtorcon.TorControlProtocol`;
see :func:`txtorcon.launch_tor` for the default one which is a
functools.partial that will call
``connect(TorProtocolFactory())`` on an appropriate
:api:`twisted.internet.endpoints.TCP4ClientEndpoint`
:param connection_creator: A no-parameter callable which
returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`
:param progress_updates: A callback which received progress
updates with three args: percent, tag, summary
:param config: a TorConfig object to connect to the
TorControlProtocl from the launched tor (should it succeed)
:param ireactortime:
An object implementing IReactorTime (i.e. a reactor) which
needs to be supplied if you pass a timeout.
:param timeout:
An int representing the timeout in seconds. If we are
unable to reach 100% by this time we will consider the
setting up of Tor to have failed. Must supply ireactortime
if you supply this.
:param kill_on_stderr:
When True, kill subprocess if we receive anything on stderr
:param stdout:
Anything subprocess writes to stdout is sent to .write() on this
:param stderr:
Anything subprocess writes to stderr is sent to .write() on this
:ivar tor_protocol: The TorControlProtocol instance connected
to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid
when the `connected_cb` callback runs.
:ivar connected_cb: Triggered when the Tor process we
represent is fully bootstrapped
"""
self.config = config
self.tor_protocol = None
self.connection_creator = connection_creator
self.progress_updates = progress_updates
self.connected_cb = defer.Deferred()
self.attempted_connect = False
self.to_delete = []
self.kill_on_stderr = kill_on_stderr
self.stderr = stderr
self.stdout = stdout
self.collected_stdout = StringIO()
self._setup_complete = False
self._did_timeout = False
self._timeout_delayed_call = None
if timeout:
if not ireactortime:
raise RuntimeError('Must supply an IReactorTime object when supplying a timeout')
ireactortime = IReactorTime(ireactortime)
self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)
def __init__(self, connection_creator, progress_updates=None, config=None,
ireactortime=None, timeout=None):
"""
This will read the output from a Tor process and attempt a
connection to its control port when it sees any 'Bootstrapped'
message on stdout. You probably don't need to use this
directly except as the return value from the
:func:`txtorcon.launch_tor` method. tor_protocol contains a
valid :class:`txtorcon.TorControlProtocol` instance by that
point.
connection_creator is a callable that should return a Deferred
that callbacks with a :class:`txtorcon.TorControlProtocol`;
see :func:`txtorcon.launch_tor` for the default one which is a
functools.partial that will call
``connect(TorProtocolFactory())`` on an appropriate
:api:`twisted.internet.endpoints.TCP4ClientEndpoint`
:param connection_creator: A no-parameter callable which
returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`
:param progress_updates: A callback which received progress
updates with three args: percent, tag, summary
:param config: a TorConfig object to connect to the
TorControlProtocl from the launched tor (should it succeed)
:param ireactortime:
An object implementing IReactorTime (i.e. a reactor) which
needs to be supplied if you pass a timeout.
:param timeout:
An int representing the timeout in seconds. If we are
unable to reach 100% by this time we will consider the
setting up of Tor to have failed. Must supply ireactortime
if you supply this.
:ivar tor_protocol: The TorControlProtocol instance connected
to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid
when the `connected_cb` callback runs.
:ivar connected_cb: Triggered when the Tor process we
represent is fully bootstrapped
"""
self.config = config
self.tor_protocol = None
self.connection_creator = connection_creator
self.progress_updates = progress_updates
self.connected_cb = defer.Deferred()
self.attempted_connect = False
self.to_delete = []
self.stderr = []
self.stdout = []
self._setup_complete = False
self._timeout_delayed_call = None
if timeout:
if not ireactortime:
raise RuntimeError('Must supply an IReactorTime object when supplying a timeout')
ireactortime = IReactorTime(ireactortime)
self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)