def runCoroutine(reactor: IReactorTime, routine: Coroutine[None, None, None] ) -> None: """Run the given coroutine, while returning control to the reactor after every processing step. Use `await asyncio.sleep(0)` in your coroutine to separate processing steps. """ try: routine.send(None) except StopIteration: pass else: reactor.callLater(0, runCoroutine, reactor, routine)
class TelemetryStore(CollectionState): """ Accepts telemetry messages and exports the accumulated information obtained from them. """ implements(ITelemetryStore) def __init__(self, time_source=the_reactor): self.__interesting_objects = CellDict(dynamic=True) CollectionState.__init__(self, self.__interesting_objects) self.__objects = {} self.__expiry_times = {} self.__time_source = IReactorTime(time_source) self.__flush_call = None # not exported def receive(self, message): """Store the supplied telemetry message object.""" message = ITelemetryMessage(message) object_id = unicode(message.get_object_id()) if object_id in self.__objects: obj = self.__objects[object_id] else: obj = self.__objects[object_id] = ITelemetryObject( # TODO: Should probably have a context object supplying last message time and delete_me() message.get_object_constructor()(object_id=object_id)) obj.receive(message) expiry = obj.get_object_expiry() self.__expiry_times[object_id] = expiry if obj.is_interesting(): self.__interesting_objects[object_id] = obj self.__maybe_schedule_flush() def __flush_expired(self): current_time = self.__time_source.seconds() deletes = [] for object_id, expiry in self.__expiry_times.iteritems(): if expiry <= current_time: deletes.append(object_id) for object_id in deletes: del self.__objects[object_id] del self.__expiry_times[object_id] if object_id in self.__interesting_objects: del self.__interesting_objects[object_id] self.__maybe_schedule_flush() def __maybe_schedule_flush(self): """Schedule a call to __flush_expired if there is not one already.""" if self.__flush_call and self.__flush_call.active(): # Could need to schedule one earlier than already scheduled. self.__flush_call.cancel() if self.__expiry_times: next_expiry = min(self.__expiry_times.itervalues()) self.__flush_call = self.__time_source.callLater( next_expiry - self.__time_source.seconds(), self.__flush_expired)
class BandwidthUpdater: def __init__(self, config, scheduler): self.bandwidth = 0 self.config = config self.scheduler = IReactorTime(scheduler) self.generator = self.next_update() def next_update(self): """ Generator that gives out the next time to do a bandwidth update, as well as what the new bandwidth value should be. Here, we toggle the bandwidth every 20 minutes. """ while True: if self.bandwidth: self.bandwidth = 0 self.burst = 0 else: self.bandwidth = 20 * 1024 * 1024 self.burst = self.bandwidth yield (datetime.datetime.now() + datetime.timedelta(minutes=20), self.bandwidth, self.burst) def do_update(self): x = self.generator.next() future = x[0] self.new_bandwidth = x[1] self.new_burst = x[2] tm = (future - datetime.datetime.now()).seconds self.scheduler.callLater(tm, self.really_update) print "waiting", tm, "seconds to adjust bandwidth" def really_update(self): print "setting bandwidth + burst to", self.new_bandwidth, self.new_burst self.config.set_config('BandWidthBurst', self.new_burst, 'BandWidthRate', self.new_bandwidth) self.doUpdate()
class BandwidthUpdater: def __init__(self, config, scheduler): self.bandwidth = 0 self.config = config self.scheduler = IReactorTime(scheduler) self.generator = self.next_update() def next_update(self): """ Generator that gives out the next time to do a bandwidth update, as well as what the new bandwidth value should be. Here, we toggle the bandwidth every 20 minutes. """ while True: if self.bandwidth: self.bandwidth = 0 self.burst = 0 else: self.bandwidth = 20 * 1024 * 1024 self.burst = self.bandwidth yield (datetime.datetime.now() + datetime.timedelta(minutes=20), self.bandwidth, self.burst) def do_update(self): x = self.generator.next() future = x[0] self.new_bandwidth = x[1] self.new_burst = x[2] tm = (future - datetime.datetime.now()).seconds self.scheduler.callLater(tm, self.really_update) print "waiting", tm, "seconds to adjust bandwidth" def really_update(self): print "setting bandwidth + burst to", self.new_bandwidth, self.new_burst self.config.set_config('BandWidthBurst', self.new_burst, 'BandWidthRate', self.new_bandwidth) self.doUpdate()
class EventualQueue(object): def __init__(self, clock): # pass clock=reactor unless you're testing self._clock = IReactorTime(clock) self._calls = [] self._flush_d = None self._timer = None def eventually(self, f, *args, **kwargs): self._calls.append((f, args, kwargs)) if not self._timer: self._timer = self._clock.callLater(0, self._turn) def fire_eventually(self, value=None): d = Deferred() self.eventually(d.callback, value) return d def _turn(self): while self._calls: (f, args, kwargs) = self._calls.pop(0) try: f(*args, **kwargs) except Exception: log.err() self._timer = None d, self._flush_d = self._flush_d, None if d: d.callback(None) def flush_sync(self): # if you have control over the Clock, this will synchronously flush the # queue assert self._clock.advance, "needs clock=twisted.internet.task.Clock()" while self._calls: self._clock.advance(0) def flush(self): # this is for unit tests, not application code assert not self._flush_d, "only one flush at a time" self._flush_d = Deferred() self.eventually(lambda: None) return self._flush_d
def deferLater( clock: IReactorTime, delay: float, callable: Optional[Callable[..., _T]] = None, *args: object, **kw: object, ) -> Deferred[_T]: """ Call the given function after a certain period of time has passed. @param clock: The object which will be used to schedule the delayed call. @param delay: The number of seconds to wait before calling the function. @param callable: The callable to call after the delay, or C{None}. @param args: The positional arguments to pass to C{callable}. @param kw: The keyword arguments to pass to C{callable}. @return: A deferred that fires with the result of the callable when the specified time has elapsed. """ def deferLaterCancel(deferred: Deferred[object]) -> None: delayedCall.cancel() def cb(result: object) -> _T: if callable is None: return None # type: ignore[return-value] return callable(*args, **kw) d: Deferred[_T] = Deferred(deferLaterCancel) d.addCallback(cb) delayedCall = clock.callLater(delay, d.callback, None) return d
class TelemetryStore(CollectionState): """ Accepts telemetry messages and exports the accumulated information obtained from them. """ def __init__(self, time_source=the_reactor): self.__interesting_objects = CellDict(dynamic=True) CollectionState.__init__(self, self.__interesting_objects) self.__objects = {} self.__expiry_times = {} self.__time_source = IReactorTime(time_source) self.__flush_call = None # not exported def receive(self, message): """Store the supplied telemetry message object.""" message = ITelemetryMessage(message) object_id = unicode(message.get_object_id()) if object_id in self.__objects: obj = self.__objects[object_id] else: obj = self.__objects[object_id] = ITelemetryObject( # TODO: Should probably have a context object supplying last message time and delete_me() message.get_object_constructor()(object_id=object_id)) obj.receive(message) expiry = obj.get_object_expiry() self.__expiry_times[object_id] = expiry if obj.is_interesting(): self.__interesting_objects[object_id] = obj self.__maybe_schedule_flush() def __flush_expired(self): current_time = self.__time_source.seconds() deletes = [] for object_id, expiry in self.__expiry_times.iteritems(): if expiry <= current_time: deletes.append(object_id) for object_id in deletes: del self.__objects[object_id] del self.__expiry_times[object_id] if object_id in self.__interesting_objects: del self.__interesting_objects[object_id] self.__maybe_schedule_flush() def __maybe_schedule_flush(self): """Schedule a call to __flush_expired if there is not one already.""" if self.__flush_call and self.__flush_call.active(): # Could need to schedule one earlier than already scheduled. self.__flush_call.cancel() if self.__expiry_times: now = self.__time_source.seconds() next_expiry = min(self.__expiry_times.itervalues()) sec_until_expiry = max(0, next_expiry - now) self.__flush_call = self.__time_source.callLater( sec_until_expiry, self.__flush_expired)
def __init__(self, connection_creator, progress_updates=None, config=None, ireactortime=None, timeout=None, kill_on_stderr=True, stdout=None, stderr=None): """ This will read the output from a Tor process and attempt a connection to its control port when it sees any 'Bootstrapped' message on stdout. You probably don't need to use this directly except as the return value from the :func:`txtorcon.launch_tor` method. tor_protocol contains a valid :class:`txtorcon.TorControlProtocol` instance by that point. connection_creator is a callable that should return a Deferred that callbacks with a :class:`txtorcon.TorControlProtocol`; see :func:`txtorcon.launch_tor` for the default one which is a functools.partial that will call ``connect(TorProtocolFactory())`` on an appropriate :api:`twisted.internet.endpoints.TCP4ClientEndpoint` :param connection_creator: A no-parameter callable which returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`. If this is None, we do NOT attempt to connect to the underlying Tor process. :param progress_updates: A callback which received progress updates with three args: percent, tag, summary :param config: a TorConfig object to connect to the TorControlProtocl from the launched tor (should it succeed) :param ireactortime: An object implementing IReactorTime (i.e. a reactor) which needs to be supplied if you pass a timeout. :param timeout: An int representing the timeout in seconds. If we are unable to reach 100% by this time we will consider the setting up of Tor to have failed. Must supply ireactortime if you supply this. :param kill_on_stderr: When True, kill subprocess if we receive anything on stderr :param stdout: Anything subprocess writes to stdout is sent to .write() on this :param stderr: Anything subprocess writes to stderr is sent to .write() on this :ivar tor_protocol: The TorControlProtocol instance connected to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid after the Deferred returned from :meth:`TorProcessProtocol.when_connected` is triggered. """ self.config = config self.tor_protocol = None self.progress_updates = progress_updates # XXX if connection_creator is not None .. is connected_cb # tied to connection_creator...? if connection_creator: self.connection_creator = connection_creator else: self.connection_creator = None # use SingleObserver self._connected_listeners = [ ] # list of Deferred (None when we're connected) self.attempted_connect = False self.to_delete = [] self.kill_on_stderr = kill_on_stderr self.stderr = stderr self.stdout = stdout self.collected_stdout = StringIO() self._setup_complete = False self._did_timeout = False self._timeout_delayed_call = None self._on_exit = [] # Deferred's we owe a call/errback to when we exit if timeout: if not ireactortime: raise RuntimeError( 'Must supply an IReactorTime object when supplying a ' 'timeout') ireactortime = IReactorTime(ireactortime) self._timeout_delayed_call = ireactortime.callLater( timeout, self._timeout_expired)
def __init__(self, connection_creator, progress_updates=None, config=None, ireactortime=None, timeout=None, kill_on_stderr=True, stdout=None, stderr=None): """ This will read the output from a Tor process and attempt a connection to its control port when it sees any 'Bootstrapped' message on stdout. You probably don't need to use this directly except as the return value from the :func:`txtorcon.launch_tor` method. tor_protocol contains a valid :class:`txtorcon.TorControlProtocol` instance by that point. connection_creator is a callable that should return a Deferred that callbacks with a :class:`txtorcon.TorControlProtocol`; see :func:`txtorcon.launch_tor` for the default one which is a functools.partial that will call ``connect(TorProtocolFactory())`` on an appropriate :api:`twisted.internet.endpoints.TCP4ClientEndpoint` :param connection_creator: A no-parameter callable which returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>` :param progress_updates: A callback which received progress updates with three args: percent, tag, summary :param config: a TorConfig object to connect to the TorControlProtocl from the launched tor (should it succeed) :param ireactortime: An object implementing IReactorTime (i.e. a reactor) which needs to be supplied if you pass a timeout. :param timeout: An int representing the timeout in seconds. If we are unable to reach 100% by this time we will consider the setting up of Tor to have failed. Must supply ireactortime if you supply this. :param kill_on_stderr: When True, kill subprocess if we receive anything on stderr :param stdout: Anything subprocess writes to stdout is sent to .write() on this :param stderr: Anything subprocess writes to stderr is sent to .write() on this :ivar tor_protocol: The TorControlProtocol instance connected to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid when the `connected_cb` callback runs. :ivar connected_cb: Triggered when the Tor process we represent is fully bootstrapped """ self.config = config self.tor_protocol = None self.connection_creator = connection_creator self.progress_updates = progress_updates self.connected_cb = defer.Deferred() self.attempted_connect = False self.to_delete = [] self.kill_on_stderr = kill_on_stderr self.stderr = stderr self.stdout = stdout self.collected_stdout = StringIO() self._setup_complete = False self._did_timeout = False self._timeout_delayed_call = None if timeout: if not ireactortime: raise RuntimeError('Must supply an IReactorTime object when supplying a timeout') ireactortime = IReactorTime(ireactortime) self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)
def __init__(self, connection_creator, progress_updates=None, config=None, ireactortime=None, timeout=None): """ This will read the output from a Tor process and attempt a connection to its control port when it sees any 'Bootstrapped' message on stdout. You probably don't need to use this directly except as the return value from the :func:`txtorcon.launch_tor` method. tor_protocol contains a valid :class:`txtorcon.TorControlProtocol` instance by that point. connection_creator is a callable that should return a Deferred that callbacks with a :class:`txtorcon.TorControlProtocol`; see :func:`txtorcon.launch_tor` for the default one which is a functools.partial that will call ``connect(TorProtocolFactory())`` on an appropriate :api:`twisted.internet.endpoints.TCP4ClientEndpoint` :param connection_creator: A no-parameter callable which returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>` :param progress_updates: A callback which received progress updates with three args: percent, tag, summary :param config: a TorConfig object to connect to the TorControlProtocl from the launched tor (should it succeed) :param ireactortime: An object implementing IReactorTime (i.e. a reactor) which needs to be supplied if you pass a timeout. :param timeout: An int representing the timeout in seconds. If we are unable to reach 100% by this time we will consider the setting up of Tor to have failed. Must supply ireactortime if you supply this. :ivar tor_protocol: The TorControlProtocol instance connected to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid when the `connected_cb` callback runs. :ivar connected_cb: Triggered when the Tor process we represent is fully bootstrapped """ self.config = config self.tor_protocol = None self.connection_creator = connection_creator self.progress_updates = progress_updates self.connected_cb = defer.Deferred() self.attempted_connect = False self.to_delete = [] self.stderr = [] self.stdout = [] self._setup_complete = False self._timeout_delayed_call = None if timeout: if not ireactortime: raise RuntimeError('Must supply an IReactorTime object when supplying a timeout') ireactortime = IReactorTime(ireactortime) self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)