class TelemetryStore(CollectionState):
"""
Accepts telemetry messages and exports the accumulated information obtained from them.
"""
implements(ITelemetryStore)
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
self.__flush_call = None
# not exported
def receive(self, message):
"""Store the supplied telemetry message object."""
message = ITelemetryMessage(message)
object_id = unicode(message.get_object_id())
if object_id in self.__objects:
obj = self.__objects[object_id]
else:
obj = self.__objects[object_id] = ITelemetryObject(
# TODO: Should probably have a context object supplying last message time and delete_me()
message.get_object_constructor()(object_id=object_id))
obj.receive(message)
expiry = obj.get_object_expiry()
self.__expiry_times[object_id] = expiry
if obj.is_interesting():
self.__interesting_objects[object_id] = obj
self.__maybe_schedule_flush()
def __flush_expired(self):
current_time = self.__time_source.seconds()
deletes = []
for object_id, expiry in self.__expiry_times.iteritems():
if expiry <= current_time:
deletes.append(object_id)
for object_id in deletes:
del self.__objects[object_id]
del self.__expiry_times[object_id]
if object_id in self.__interesting_objects:
del self.__interesting_objects[object_id]
self.__maybe_schedule_flush()
def __maybe_schedule_flush(self):
"""Schedule a call to __flush_expired if there is not one already."""
if self.__flush_call and self.__flush_call.active():
# Could need to schedule one earlier than already scheduled.
self.__flush_call.cancel()
if self.__expiry_times:
next_expiry = min(self.__expiry_times.itervalues())
self.__flush_call = self.__time_source.callLater(
next_expiry - self.__time_source.seconds(),
self.__flush_expired)
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
self.__flush_call = None
def __init__(self, time_source=the_reactor):
self.__interesting_objects = {}
CollectionState.__init__(self,
self.__interesting_objects,
dynamic=True)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
def __init__(self, reactor, host, port, endpoint, timestamps=None):
self._host = host
self._port = port
self._reactor = reactor
self._endpoint = endpoint
self._timestamps = None
self._timer = None
if timestamps is not None:
self._timestamps = timestamps
self._timer = IReactorTime(reactor)
def __init__(self, reactor, endpoint, proxy_config, timestamps=None):
self._host = proxy_config['host']
self._port = proxy_config['port']
self._proxy_config = proxy_config
self._reactor = reactor
self._endpoint = endpoint
self._timestamps = None
self._timer = None
if timestamps is not None:
self._timestamps = timestamps
self._timer = IReactorTime(reactor)
def runCoroutine(reactor: IReactorTime,
routine: Coroutine[None, None, None]
) -> None:
"""Run the given coroutine, while returning control to the reactor
after every processing step.
Use `await asyncio.sleep(0)` in your coroutine to separate processing
steps.
"""
try:
routine.send(None)
except StopIteration:
pass
else:
reactor.callLater(0, runCoroutine, reactor, routine)
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
self.__flush_call = None
def test_expires_with_update(self):
"""
This test updates the expiry time and checks that we properly
delay our expiry callback.
"""
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
# now do an actual update to an existing Addr entry.
now = datetime.datetime.now() + datetime.timedelta(seconds=10)
nowutc = datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
line = 'www.example.com 72.30.2.43 "%s" EXPIRES="%s"' % (now.strftime(
self.fmt), nowutc.strftime(self.fmt))
am.update(line)
self.assertTrue(am.find('www.example.com'))
# the update
now = datetime.datetime.now() + datetime.timedelta(seconds=20)
nowutc = datetime.datetime.utcnow() + datetime.timedelta(seconds=20)
line = 'www.example.com 72.30.2.43 "%s" EXPIRES="%s"' % (now.strftime(
self.fmt), nowutc.strftime(self.fmt))
am.update(line)
self.assertTrue('www.example.com' in am.addr)
# advance time by the old expiry value and we should still
# find the entry
clock.advance(10)
self.assertTrue('www.example.com' in am.addr)
# ...but advance past the new expiry (another 10 seconds) and
# it should vanish
clock.advance(10)
self.assertTrue('www.example.com' not in am.addr)
def test_listeners(self):
self.expires = []
self.addrmap = []
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
am.add_listener(self)
now = datetime.datetime.now() + datetime.timedelta(seconds=10)
nowutc = datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
line = 'www.example.com 72.30.2.43 "%s" EXPIRES="%s"' % (now.strftime(
self.fmt), nowutc.strftime(self.fmt))
am.update(line)
# see if our listener got an update
a = am.find('www.example.com')
self.assertEqual(self.addrmap, [a])
# advance time past when the expiry should have occurred
clock.advance(10)
# check that our listener got an expires event
self.assertEqual(self.expires, ['www.example.com'])
async def _handle_json_response(
reactor: IReactorTime,
timeout_sec: float,
request: MatrixFederationRequest,
response: IResponse,
start_ms: int,
):
"""
Reads the JSON body of a response, with a timeout
Args:
reactor: twisted reactor, for the timeout
timeout_sec: number of seconds to wait for response to complete
request: the request that triggered the response
response: response to the request
start_ms: Timestamp when request was made
Returns:
dict: parsed JSON response
"""
try:
check_content_type_is_json(response.headers)
d = treq.json_content(response)
d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor)
body = await make_deferred_yieldable(d)
except TimeoutError as e:
logger.warning(
"{%s} [%s] Timed out reading response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except Exception as e:
logger.warning(
"{%s} [%s] Error reading response %s %s: %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
e,
)
raise
time_taken_secs = reactor.seconds() - start_ms / 1000
logger.info(
"{%s} [%s] Completed request: %d %s in %.2f secs - %s %s",
request.txn_id,
request.destination,
response.code,
response.phrase.decode("ascii", errors="replace"),
time_taken_secs,
request.method,
request.uri.decode("ascii"),
)
return body
def __init__(self, reactor):
super(TwistedRunLoop, self).__init__()
if not IReactorCore.providedBy(reactor):
raise TypeError('reactor must implement IReactorCore')
if not IReactorTime.providedBy(reactor):
raise TypeError('reactor must implement IReactorTime')
self.__reactor = reactor
# TODO(strager): Make thread-safe.
self.__delayed_calls = []
class SOCKSWrapper(object):
implements(IStreamClientEndpoint)
factory = SOCKSv4ClientFactory
def __init__(self, reactor, host, port, endpoint, timestamps=None):
self._host = host
self._port = port
self._reactor = reactor
self._endpoint = endpoint
self._timestamps = None
self._timer = None
if timestamps is not None:
self._timestamps = timestamps
self._timer = IReactorTime(reactor)
def noteTime(self, event):
if self._timer:
self._timestamps[event] = self._timer.seconds()
def connect(self, protocolFactory):
"""
Return a deferred firing when the SOCKS connection is established.
"""
def createWrappingFactory(f):
"""
Wrap creation of _WrappingFactory since __init__() doesn't
take a canceller as of Twisted 12.1 or something.
"""
if len(inspect.getargspec(_WrappingFactory.__init__)[0]) == 3:
def _canceller(deferred):
connector.stopConnecting()
deferred.errback(
error.ConnectingCancelledError(
connector.getDestination()))
return _WrappingFactory(f, _canceller)
else: # Twisted >= 12.1.
return _WrappingFactory(f)
self.noteTime('START')
try:
# Connect with an intermediate SOCKS factory/protocol,
# which then hands control to the provided protocolFactory
# once a SOCKS connection has been established.
f = self.factory()
f.postHandshakeEndpoint = self._endpoint
f.postHandshakeFactory = protocolFactory
f.handshakeDone = defer.Deferred()
f._timestamps = self._timestamps
f._timer = self._timer
wf = createWrappingFactory(f)
self._reactor.connectTCP(self._host, self._port, wf)
self.noteTime('SOCKET')
return f.handshakeDone
except:
return defer.fail()
class SOCKSWrapper(object):
implements(IStreamClientEndpoint)
factory = SOCKSv4ClientFactory
def __init__(self, reactor, host, port, endpoint, timestamps=None):
self._host = host
self._port = port
self._reactor = reactor
self._endpoint = endpoint
self._timestamps = None
self._timer = None
if timestamps is not None:
self._timestamps = timestamps
self._timer = IReactorTime(reactor)
def noteTime(self, event):
if self._timer:
self._timestamps[event] = self._timer.seconds()
def connect(self, protocolFactory):
"""
Return a deferred firing when the SOCKS connection is established.
"""
def createWrappingFactory(f):
"""
Wrap creation of _WrappingFactory since __init__() doesn't
take a canceller as of Twisted 12.1 or something.
"""
if len(inspect.getargspec(_WrappingFactory.__init__)[0]) == 3:
def _canceller(deferred):
connector.stopConnecting()
deferred.errback(error.ConnectingCancelledError(connector.getDestination()))
return _WrappingFactory(f, _canceller)
else: # Twisted >= 12.1.
return _WrappingFactory(f)
self.noteTime("START")
try:
# Connect with an intermediate SOCKS factory/protocol,
# which then hands control to the provided protocolFactory
# once a SOCKS connection has been established.
f = self.factory()
f.postHandshakeEndpoint = self._endpoint
f.postHandshakeFactory = protocolFactory
f.handshakeDone = defer.Deferred()
f._timestamps = self._timestamps
f._timer = self._timer
wf = createWrappingFactory(f)
self._reactor.connectTCP(self._host, self._port, wf)
self.noteTime("SOCKET")
return f.handshakeDone
except:
return defer.fail()
class EventualQueue(object):
def __init__(self, clock):
# pass clock=reactor unless you're testing
self._clock = IReactorTime(clock)
self._calls = []
self._flush_d = None
self._timer = None
def eventually(self, f, *args, **kwargs):
self._calls.append((f, args, kwargs))
if not self._timer:
self._timer = self._clock.callLater(0, self._turn)
def fire_eventually(self, value=None):
d = Deferred()
self.eventually(d.callback, value)
return d
def _turn(self):
while self._calls:
(f, args, kwargs) = self._calls.pop(0)
try:
f(*args, **kwargs)
except Exception:
log.err()
self._timer = None
d, self._flush_d = self._flush_d, None
if d:
d.callback(None)
def flush_sync(self):
# if you have control over the Clock, this will synchronously flush the
# queue
assert self._clock.advance, "needs clock=twisted.internet.task.Clock()"
while self._calls:
self._clock.advance(0)
def flush(self):
# this is for unit tests, not application code
assert not self._flush_d, "only one flush at a time"
self._flush_d = Deferred()
self.eventually(lambda: None)
return self._flush_d
def __init__(self, reactor, host, port, endpoint, timestamps=None):
self._host = host
self._port = port
self._reactor = reactor
self._endpoint = endpoint
self._timestamps = None
self._timer = None
if timestamps is not None:
self._timestamps = timestamps
self._timer = IReactorTime(reactor)
def test_8596_cached_1(self):
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
line = 'example.com 192.0.2.1 NEVER CACHED="YES"'
am.update(line)
self.assertTrue('example.com' in am.addr)
self.assertEqual(len(clock.getDelayedCalls()), 0)
def test_8596_cached_2(self):
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
line = 'example.com 192.0.43.10 "2013-04-03 22:29:11" EXPIRES="2013-04-03 20:29:11" CACHED="NO"'
am.update(line)
self.assertTrue('example.com' in am.addr)
self.assertEqual(len(clock.getDelayedCalls()), 1)
def test_8596_cached_3(self):
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
line = 'example.invalid <error> "2013-04-03 08:28:52" error=yes EXPIRES="2013-04-03 06:28:52" CACHE="NO"'
am.update(line)
self.assertTrue('example.invalid' not in am.addr)
self.assertEqual(len(clock.getDelayedCalls()), 0)
class BandwidthUpdater:
def __init__(self, config, scheduler):
self.bandwidth = 0
self.config = config
self.scheduler = IReactorTime(scheduler)
self.generator = self.next_update()
def next_update(self):
"""
Generator that gives out the next time to do a bandwidth update,
as well as what the new bandwidth value should be. Here, we toggle
the bandwidth every 20 minutes.
"""
while True:
if self.bandwidth:
self.bandwidth = 0
self.burst = 0
else:
self.bandwidth = 20 * 1024 * 1024
self.burst = self.bandwidth
yield (datetime.datetime.now() + datetime.timedelta(minutes=20),
self.bandwidth, self.burst)
def do_update(self):
x = self.generator.next()
future = x[0]
self.new_bandwidth = x[1]
self.new_burst = x[2]
tm = (future - datetime.datetime.now()).seconds
self.scheduler.callLater(tm, self.really_update)
print "waiting", tm, "seconds to adjust bandwidth"
def really_update(self):
print "setting bandwidth + burst to", self.new_bandwidth, self.new_burst
self.config.set_config('BandWidthBurst', self.new_burst,
'BandWidthRate', self.new_bandwidth)
self.doUpdate()
class BandwidthUpdater:
def __init__(self, config, scheduler):
self.bandwidth = 0
self.config = config
self.scheduler = IReactorTime(scheduler)
self.generator = self.next_update()
def next_update(self):
"""
Generator that gives out the next time to do a bandwidth update,
as well as what the new bandwidth value should be. Here, we toggle
the bandwidth every 20 minutes.
"""
while True:
if self.bandwidth:
self.bandwidth = 0
self.burst = 0
else:
self.bandwidth = 20 * 1024 * 1024
self.burst = self.bandwidth
yield (datetime.datetime.now() + datetime.timedelta(minutes=20),
self.bandwidth, self.burst)
def do_update(self):
x = self.generator.next()
future = x[0]
self.new_bandwidth = x[1]
self.new_burst = x[2]
tm = (future - datetime.datetime.now()).seconds
self.scheduler.callLater(tm, self.really_update)
print "waiting", tm, "seconds to adjust bandwidth"
def really_update(self):
print "setting bandwidth + burst to", self.new_bandwidth, self.new_burst
self.config.set_config('BandWidthBurst', self.new_burst,
'BandWidthRate', self.new_bandwidth)
self.doUpdate()
def test_expires_never(self):
"""
Test a NEVER expires line, as in what we'd get a startup for a
configured address-mapping.
"""
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
line = 'www.example.com 72.30.2.43 "NEVER"'
am.update(line)
self.assertTrue('www.example.com' in am.addr)
self.assertEqual(len(clock.getDelayedCalls()), 0)
class TelemetryStore(CollectionState):
"""
Accepts telemetry messages and exports the accumulated information obtained from them.
"""
implements(ITelemetryStore)
def __init__(self, time_source=the_reactor):
self.__interesting_objects = {}
CollectionState.__init__(self,
self.__interesting_objects,
dynamic=True)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
# not exported
def receive(self, message):
"""Store the supplied telemetry message object."""
message = ITelemetryMessage(message)
object_id = unicode(message.get_object_id())
if object_id in self.__objects:
obj = self.__objects[object_id]
else:
obj = self.__objects[object_id] = ITelemetryObject(
# TODO: Should probably have a context object supplying last message time and delete_me()
message.get_object_constructor()(object_id=object_id))
obj.receive(message)
self.__expiry_times[object_id] = obj.get_object_expiry()
if obj.is_interesting():
self.__interesting_objects[object_id] = obj
# logically independent but this is a convenient time, and this approach allows us to borrow the receive time rather than reading the system clock ourselves.
# TODO: But it doesn't deal with timing out when we aren't receiving messages
self.__flush_expired()
def __flush_expired(self):
current_time = self.__time_source.seconds()
deletes = []
for object_id, expiry in self.__expiry_times.iteritems():
if expiry <= current_time:
deletes.append(object_id)
for object_id in deletes:
del self.__objects[object_id]
del self.__expiry_times[object_id]
if object_id in self.__interesting_objects:
del self.__interesting_objects[object_id]
class TelemetryStore(CollectionState):
"""
Accepts telemetry messages and exports the accumulated information obtained from them.
"""
implements(ITelemetryStore)
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
# not exported
def receive(self, message):
"""Store the supplied telemetry message object."""
message = ITelemetryMessage(message)
object_id = unicode(message.get_object_id())
if object_id in self.__objects:
obj = self.__objects[object_id]
else:
obj = self.__objects[object_id] = ITelemetryObject(
# TODO: Should probably have a context object supplying last message time and delete_me()
message.get_object_constructor()(object_id=object_id))
obj.receive(message)
self.__expiry_times[object_id] = obj.get_object_expiry()
if obj.is_interesting():
self.__interesting_objects[object_id] = obj
# logically independent but this is a convenient time, and this approach allows us to borrow the receive time rather than reading the system clock ourselves.
# TODO: But it doesn't deal with timing out when we aren't receiving messages
self.__flush_expired()
def __flush_expired(self):
current_time = self.__time_source.seconds()
deletes = []
for object_id, expiry in self.__expiry_times.iteritems():
if expiry <= current_time:
deletes.append(object_id)
for object_id in deletes:
del self.__objects[object_id]
del self.__expiry_times[object_id]
if object_id in self.__interesting_objects:
del self.__interesting_objects[object_id]
def test_expires(self):
"""
Test simply expiry case
"""
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
now = datetime.datetime.now() + datetime.timedelta(seconds=10)
nowutc = datetime.datetime.utcnow() + datetime.timedelta(seconds=10)
line = 'www.example.com 72.30.2.43 "%s" EXPIRES="%s"' % (now.strftime(self.fmt), nowutc.strftime(self.fmt))
am.update(line)
self.assertTrue('www.example.com' in am.addr)
# advance time past when the expiry should have occurred
clock.advance(10)
self.assertTrue('www.example.com' not in am.addr)
def test_expires_old(self):
"""
Test something that expires before "now"
"""
clock = task.Clock()
am = AddrMap()
am.scheduler = IReactorTime(clock)
now = datetime.datetime.now() + datetime.timedelta(seconds=-10)
nowutc = datetime.datetime.utcnow() + datetime.timedelta(seconds=-10)
line = 'www.example.com 72.30.2.43 "%s" EXPIRES="%s"' % (now.strftime(self.fmt), nowutc.strftime(self.fmt))
am.update(line)
self.assertTrue('www.example.com' in am.addr)
# arguably we shouldn't even have put this in the map maybe,
# but the reactor needs to iterate before our expiry callback
# gets called (right away) which is simulated by the
# clock.advance call
clock.advance(0)
self.assertTrue('www.example.com' not in am.addr)
def deferLater(
clock: IReactorTime,
delay: float,
callable: Optional[Callable[..., _T]] = None,
*args: object,
**kw: object,
) -> Deferred[_T]:
"""
Call the given function after a certain period of time has passed.
@param clock: The object which will be used to schedule the delayed
call.
@param delay: The number of seconds to wait before calling the function.
@param callable: The callable to call after the delay, or C{None}.
@param args: The positional arguments to pass to C{callable}.
@param kw: The keyword arguments to pass to C{callable}.
@return: A deferred that fires with the result of the callable when the
specified time has elapsed.
"""
def deferLaterCancel(deferred: Deferred[object]) -> None:
delayedCall.cancel()
def cb(result: object) -> _T:
if callable is None:
return None # type: ignore[return-value]
return callable(*args, **kw)
d: Deferred[_T] = Deferred(deferLaterCancel)
d.addCallback(cb)
delayedCall = clock.callLater(delay, d.callback, None)
return d
def __init__(self):
self.addr = {}
self.scheduler = IReactorTime(reactor)
self.listeners = []
async def _handle_response(
reactor: IReactorTime,
timeout_sec: float,
request: MatrixFederationRequest,
response: IResponse,
start_ms: int,
parser: ByteParser[T],
max_response_size: Optional[int] = None,
) -> T:
"""
Reads the body of a response with a timeout and sends it to a parser
Args:
reactor: twisted reactor, for the timeout
timeout_sec: number of seconds to wait for response to complete
request: the request that triggered the response
response: response to the request
start_ms: Timestamp when request was made
parser: The parser for the response
max_response_size: The maximum size to read from the response, if None
uses the default.
Returns:
The parsed response
"""
if max_response_size is None:
max_response_size = MAX_RESPONSE_SIZE
try:
check_content_type_is(response.headers, parser.CONTENT_TYPE)
d = read_body_with_max_size(response, parser, max_response_size)
d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor)
length = await make_deferred_yieldable(d)
value = parser.finish()
except BodyExceededMaxSize as e:
# The response was too big.
logger.warning(
"{%s} [%s] JSON response exceeded max size %i - %s %s",
request.txn_id,
request.destination,
MAX_RESPONSE_SIZE,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=False) from e
except ValueError as e:
# The content was invalid.
logger.warning(
"{%s} [%s] Failed to parse response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=False) from e
except defer.TimeoutError as e:
logger.warning(
"{%s} [%s] Timed out reading response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except ResponseFailed as e:
logger.warning(
"{%s} [%s] Failed to read response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except Exception as e:
logger.warning(
"{%s} [%s] Error reading response %s %s: %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
e,
)
raise
time_taken_secs = reactor.seconds() - start_ms / 1000
logger.info(
"{%s} [%s] Completed request: %d %s in %.2f secs, got %d bytes - %s %s",
request.txn_id,
request.destination,
response.code,
response.phrase.decode("ascii", errors="replace"),
time_taken_secs,
length,
request.method,
request.uri.decode("ascii"),
)
return value
def __init__(self, time_source=the_reactor):
self.__interesting_objects = {}
CollectionState.__init__(self, self.__interesting_objects, dynamic=True)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
def __init__(self, clock):
# pass clock=reactor unless you're testing
self._clock = IReactorTime(clock)
self._calls = []
self._flush_d = None
self._timer = None
def __init__(self, config, scheduler):
self.bandwidth = 0
self.config = config
self.scheduler = IReactorTime(scheduler)
self.generator = self.next_update()
def __init__(self, connection_creator, progress_updates=None, config=None,
ireactortime=None, timeout=None, kill_on_stderr=True,
stdout=None, stderr=None):
"""
This will read the output from a Tor process and attempt a
connection to its control port when it sees any 'Bootstrapped'
message on stdout. You probably don't need to use this
directly except as the return value from the
:func:`txtorcon.launch_tor` method. tor_protocol contains a
valid :class:`txtorcon.TorControlProtocol` instance by that
point.
connection_creator is a callable that should return a Deferred
that callbacks with a :class:`txtorcon.TorControlProtocol`;
see :func:`txtorcon.launch_tor` for the default one which is a
functools.partial that will call
``connect(TorProtocolFactory())`` on an appropriate
:api:`twisted.internet.endpoints.TCP4ClientEndpoint`
:param connection_creator: A no-parameter callable which
returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`
:param progress_updates: A callback which received progress
updates with three args: percent, tag, summary
:param config: a TorConfig object to connect to the
TorControlProtocl from the launched tor (should it succeed)
:param ireactortime:
An object implementing IReactorTime (i.e. a reactor) which
needs to be supplied if you pass a timeout.
:param timeout:
An int representing the timeout in seconds. If we are
unable to reach 100% by this time we will consider the
setting up of Tor to have failed. Must supply ireactortime
if you supply this.
:param kill_on_stderr:
When True, kill subprocess if we receive anything on stderr
:param stdout:
Anything subprocess writes to stdout is sent to .write() on this
:param stderr:
Anything subprocess writes to stderr is sent to .write() on this
:ivar tor_protocol: The TorControlProtocol instance connected
to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid
when the `connected_cb` callback runs.
:ivar connected_cb: Triggered when the Tor process we
represent is fully bootstrapped
"""
self.config = config
self.tor_protocol = None
self.connection_creator = connection_creator
self.progress_updates = progress_updates
self.connected_cb = defer.Deferred()
self.attempted_connect = False
self.to_delete = []
self.kill_on_stderr = kill_on_stderr
self.stderr = stderr
self.stdout = stdout
self.collected_stdout = StringIO()
self._setup_complete = False
self._did_timeout = False
self._timeout_delayed_call = None
if timeout:
if not ireactortime:
raise RuntimeError('Must supply an IReactorTime object when supplying a timeout')
ireactortime = IReactorTime(ireactortime)
self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)
def call_later(delay, fun, *args, **kwargs):
return IReactorTime(_get_loop()).callLater(delay, fun, *args, **kwargs)
def __init__(self, config, scheduler):
self.bandwidth = 0
self.config = config
self.scheduler = IReactorTime(scheduler)
self.generator = self.next_update()
def __init__(self, connection_creator, progress_updates=None, config=None,
ireactortime=None, timeout=None):
"""
This will read the output from a Tor process and attempt a
connection to its control port when it sees any 'Bootstrapped'
message on stdout. You probably don't need to use this
directly except as the return value from the
:func:`txtorcon.launch_tor` method. tor_protocol contains a
valid :class:`txtorcon.TorControlProtocol` instance by that
point.
connection_creator is a callable that should return a Deferred
that callbacks with a :class:`txtorcon.TorControlProtocol`;
see :func:`txtorcon.launch_tor` for the default one which is a
functools.partial that will call
``connect(TorProtocolFactory())`` on an appropriate
:api:`twisted.internet.endpoints.TCP4ClientEndpoint`
:param connection_creator: A no-parameter callable which
returns a Deferred which promises a :api:`twisted.internet.interfaces.IStreamClientEndpoint <IStreamClientEndpoint>`
:param progress_updates: A callback which received progress
updates with three args: percent, tag, summary
:param config: a TorConfig object to connect to the
TorControlProtocl from the launched tor (should it succeed)
:param ireactortime:
An object implementing IReactorTime (i.e. a reactor) which
needs to be supplied if you pass a timeout.
:param timeout:
An int representing the timeout in seconds. If we are
unable to reach 100% by this time we will consider the
setting up of Tor to have failed. Must supply ireactortime
if you supply this.
:ivar tor_protocol: The TorControlProtocol instance connected
to the Tor this :api:`twisted.internet.protocol.ProcessProtocol <ProcessProtocol>`` is speaking to. Will be valid
when the `connected_cb` callback runs.
:ivar connected_cb: Triggered when the Tor process we
represent is fully bootstrapped
"""
self.config = config
self.tor_protocol = None
self.connection_creator = connection_creator
self.progress_updates = progress_updates
self.connected_cb = defer.Deferred()
self.attempted_connect = False
self.to_delete = []
self.stderr = []
self.stdout = []
self._setup_complete = False
self._timeout_delayed_call = None
if timeout:
if not ireactortime:
raise RuntimeError('Must supply an IReactorTime object when supplying a timeout')
ireactortime = IReactorTime(ireactortime)
self._timeout_delayed_call = ireactortime.callLater(timeout, self.timeout_expired)
class TelemetryStore(CollectionState):
"""
Accepts telemetry messages and exports the accumulated information obtained from them.
"""
def __init__(self, time_source=the_reactor):
self.__interesting_objects = CellDict(dynamic=True)
CollectionState.__init__(self, self.__interesting_objects)
self.__objects = {}
self.__expiry_times = {}
self.__time_source = IReactorTime(time_source)
self.__flush_call = None
# not exported
def receive(self, message):
"""Store the supplied telemetry message object."""
message = ITelemetryMessage(message)
object_id = unicode(message.get_object_id())
if object_id in self.__objects:
obj = self.__objects[object_id]
else:
obj = self.__objects[object_id] = ITelemetryObject(
# TODO: Should probably have a context object supplying last message time and delete_me()
message.get_object_constructor()(object_id=object_id))
obj.receive(message)
expiry = obj.get_object_expiry()
self.__expiry_times[object_id] = expiry
if obj.is_interesting():
self.__interesting_objects[object_id] = obj
self.__maybe_schedule_flush()
def __flush_expired(self):
current_time = self.__time_source.seconds()
deletes = []
for object_id, expiry in self.__expiry_times.iteritems():
if expiry <= current_time:
deletes.append(object_id)
for object_id in deletes:
del self.__objects[object_id]
del self.__expiry_times[object_id]
if object_id in self.__interesting_objects:
del self.__interesting_objects[object_id]
self.__maybe_schedule_flush()
def __maybe_schedule_flush(self):
"""Schedule a call to __flush_expired if there is not one already."""
if self.__flush_call and self.__flush_call.active():
# Could need to schedule one earlier than already scheduled.
self.__flush_call.cancel()
if self.__expiry_times:
now = self.__time_source.seconds()
next_expiry = min(self.__expiry_times.itervalues())
sec_until_expiry = max(0, next_expiry - now)
self.__flush_call = self.__time_source.callLater(
sec_until_expiry,
self.__flush_expired)
def __init__(self,
connection_creator,
progress_updates=None,
config=None,
ireactortime=None,
timeout=None,
kill_on_stderr=True,
stdout=None,
stderr=None):
"""
This will read the output from a Tor process and attempt a
connection to its control port when it sees any 'Bootstrapped'
message on stdout. You probably don't need to use this
directly except as the return value from the
:func:`txtorcon.launch_tor` method. tor_protocol contains a
valid :class:`txtorcon.TorControlProtocol` instance by that
point.
connection_creator is a callable that should return a Deferred
that callbacks with a :class:`txtorcon.TorControlProtocol`;
see :func:`txtorcon.launch_tor` for the default one which is a
functools.partial that will call
``connect(TorProtocolFactory())`` on an appropriate
:api:`twisted.internet.endpoints.TCP4ClientEndpoint`
:param connection_creator: A no-parameter callable which
returns a Deferred which promises a
:api:`twisted.internet.interfaces.IStreamClientEndpoint
<IStreamClientEndpoint>`. If this is None, we do NOT
attempt to connect to the underlying Tor process.
:param progress_updates: A callback which received progress
updates with three args: percent, tag, summary
:param config: a TorConfig object to connect to the
TorControlProtocl from the launched tor (should it succeed)
:param ireactortime:
An object implementing IReactorTime (i.e. a reactor) which
needs to be supplied if you pass a timeout.
:param timeout:
An int representing the timeout in seconds. If we are
unable to reach 100% by this time we will consider the
setting up of Tor to have failed. Must supply ireactortime
if you supply this.
:param kill_on_stderr:
When True, kill subprocess if we receive anything on stderr
:param stdout:
Anything subprocess writes to stdout is sent to .write() on this
:param stderr:
Anything subprocess writes to stderr is sent to .write() on this
:ivar tor_protocol: The TorControlProtocol instance connected
to the Tor this
:api:`twisted.internet.protocol.ProcessProtocol
<ProcessProtocol>`` is speaking to. Will be valid after
the Deferred returned from
:meth:`TorProcessProtocol.when_connected` is triggered.
"""
self.config = config
self.tor_protocol = None
self.progress_updates = progress_updates
# XXX if connection_creator is not None .. is connected_cb
# tied to connection_creator...?
if connection_creator:
self.connection_creator = connection_creator
else:
self.connection_creator = None
# use SingleObserver
self._connected_listeners = [
] # list of Deferred (None when we're connected)
self.attempted_connect = False
self.to_delete = []
self.kill_on_stderr = kill_on_stderr
self.stderr = stderr
self.stdout = stdout
self.collected_stdout = StringIO()
self._setup_complete = False
self._did_timeout = False
self._timeout_delayed_call = None
self._on_exit = [] # Deferred's we owe a call/errback to when we exit
if timeout:
if not ireactortime:
raise RuntimeError(
'Must supply an IReactorTime object when supplying a '
'timeout')
ireactortime = IReactorTime(ireactortime)
self._timeout_delayed_call = ireactortime.callLater(
timeout, self._timeout_expired)
async def _handle_json_response(
reactor: IReactorTime,
timeout_sec: float,
request: MatrixFederationRequest,
response: IResponse,
start_ms: int,
) -> JsonDict:
"""
Reads the JSON body of a response, with a timeout
Args:
reactor: twisted reactor, for the timeout
timeout_sec: number of seconds to wait for response to complete
request: the request that triggered the response
response: response to the request
start_ms: Timestamp when request was made
Returns:
The parsed JSON response
"""
try:
check_content_type_is_json(response.headers)
buf = StringIO()
d = read_body_with_max_size(response, BinaryIOWrapper(buf),
MAX_RESPONSE_SIZE)
d = timeout_deferred(d, timeout=timeout_sec, reactor=reactor)
def parse(_len: int):
return json_decoder.decode(buf.getvalue())
d.addCallback(parse)
body = await make_deferred_yieldable(d)
except BodyExceededMaxSize as e:
# The response was too big.
logger.warning(
"{%s} [%s] JSON response exceeded max size %i - %s %s",
request.txn_id,
request.destination,
MAX_RESPONSE_SIZE,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=False) from e
except ValueError as e:
# The JSON content was invalid.
logger.warning(
"{%s} [%s] Failed to parse JSON response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=False) from e
except defer.TimeoutError as e:
logger.warning(
"{%s} [%s] Timed out reading response - %s %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
)
raise RequestSendFailed(e, can_retry=True) from e
except Exception as e:
logger.warning(
"{%s} [%s] Error reading response %s %s: %s",
request.txn_id,
request.destination,
request.method,
request.uri.decode("ascii"),
e,
)
raise
time_taken_secs = reactor.seconds() - start_ms / 1000
logger.info(
"{%s} [%s] Completed request: %d %s in %.2f secs - %s %s",
request.txn_id,
request.destination,
response.code,
response.phrase.decode("ascii", errors="replace"),
time_taken_secs,
request.method,
request.uri.decode("ascii"),
)
return body
