def test_diagnose_unwraps_first_error_if_apierr_or_connection_error(self):
"""
:func:`diagnose` unwraps :class:`FirstError`, no matter how deeply
nested, and wraps the underlying :class:`ConnectionRefusedError` and
:class:`APIError` in an :class:`UpstreamError`
"""
def _wrap(exception):
return lambda: fail(
FirstError(
Failure(FirstError(Failure(exception), 0)),
0))
f = self.failureResultOf(
diagnose("system", "operation")(_wrap(APIError(200, {})))(),
UpstreamError)
self.assertTrue(f.value.reason.check(APIError))
self.assertEqual(f.value.system, "system")
self.assertEqual(f.value.operation, "operation")
f = self.failureResultOf(
diagnose("system", "operation")(
_wrap(ConnectionRefusedError('meh')))(),
UpstreamError)
self.assertTrue(f.value.reason.check(ConnectionRefusedError))
self.assertEqual(f.value.system, "system")
self.assertEqual(f.value.operation, "operation")
def test_connect_error(self, *args):
'''
Ensure the runner doesn't swallow errors and that it exits the
reactor properly if there is one.
'''
try:
from autobahn.twisted.wamp import ApplicationRunner
from twisted.internet.error import ConnectionRefusedError
# the 'reactor' member doesn't exist until we import it
from twisted.internet import reactor # noqa: F401
except ImportError:
raise unittest.SkipTest('No twisted')
runner = ApplicationRunner(u'ws://localhost:1', u'realm')
exception = ConnectionRefusedError("It's a trap!")
with patch('twisted.internet.reactor',
FakeReactor(exception)) as mockreactor:
self.assertRaises(
ConnectionRefusedError,
# pass a no-op session-creation method
runner.run,
lambda _: None,
start_reactor=True)
self.assertTrue(mockreactor.stop_called)
def test_updateMetadata_retry(self):
"""
Updating the broker metadata of the client changes the destination of
the next connection attempt. Any outstanding connections remain until
then.
"""
d = self.brokerClient.makeRequest(1, METADATA_REQUEST_1)
self.assertNoResult(d)
self.assertEqual([('host', 1234)], self.connections.calls)
self.brokerClient.updateMetadata(
BrokerMetadata(node_id=1, host='other', port=2345))
self.assertEqual('other', self.brokerClient.host)
self.assertEqual(2345, self.brokerClient.port)
# A connection to the new host was *not* triggered.
self.assertEqual([('host', 1234)], self.connections.calls)
# Fail the pending connection:
self.connections.fail('host', ConnectionRefusedError("Nope."))
# Trigger retry attempt, which happens after a delay:
self.reactor.advance(self.retryDelay)
# The retry attempt was made to the new host.
self.assertEqual([('host', 1234), ('other', 2345)],
self.connections.calls)
def test_reentrantTCPQueryErrbackOnConnectionFailure(self):
"""
An errback on the deferred returned by
L{client.Resolver.queryTCP} may trigger another TCP query.
"""
reactor = proto_helpers.MemoryReactor()
resolver = client.Resolver(
servers=[('127.0.0.1', 10053)],
reactor=reactor)
q = dns.Query('example.com')
# First query sent
d = resolver.queryTCP(q)
# Repeat the query when the first query fails
def reissue(e):
e.trap(ConnectionRefusedError)
return resolver.queryTCP(q)
d.addErrback(reissue)
self.assertEqual(len(reactor.tcpClients), 1)
self.assertEqual(len(reactor.connectors), 1)
host, port, factory, timeout, bindAddress = reactor.tcpClients[0]
# First query fails
f1 = failure.Failure(ConnectionRefusedError())
factory.clientConnectionFailed(
reactor.connectors[0],
f1)
# A second TCP connection is immediately attempted
self.assertEqual(len(reactor.tcpClients), 2)
self.assertEqual(len(reactor.connectors), 2)
# No result expected until the second chained query returns
self.assertNoResult(d)
# Second query fails
f2 = failure.Failure(ConnectionRefusedError())
factory.clientConnectionFailed(
reactor.connectors[1],
f2)
# Original deferred now fires with the second failure
f = self.failureResultOf(d, ConnectionRefusedError)
self.assertIdentical(f, f2)
def reject_connection(self, reason=None):
"""
Reject a pending connection.
"""
assert self.pending, "Connection is not pending."
if reason is None:
reason = ConnectionRefusedError()
self._accept_d.errback(reason)
def test_ebRequest(self):
"""Send a :api:`twisted.python.failure.Failure` and check that the
resulting protocol is a
:class:`~bridgedb.txrecaptcha.RecaptchaResponseProtocol`.
"""
msg = "Einhorn"
fail = failure.Failure(ConnectionRefusedError(msg))
result = txrecaptcha._ebRequest(fail)
self.assertIsInstance(result, txrecaptcha.RecaptchaResponse)
self.assertRegexpMatches(result.error_code, msg)
def failOnce(self, reason=Failure(ConnectionRefusedError())):
"""
Fail a single TCP connection established on this
L{ConnectionCompleter}'s L{MemoryReactor}.
@param reason: the reason to provide that the connection failed.
@type reason: L{Failure}
"""
self._reactor.tcpClients.pop(0)[2].clientConnectionFailed(
self._reactor.connectors.pop(0), reason)
def reject_connection(self):
"""
Reject a pending connection.
This is only useful if auto-accept is disabled. The deferred returned
by waiting for `await_connected()` for this connection will never fire.
"""
assert self.has_pending_connection(), "No pending connection."
self._accept_d.errback(ConnectionRefusedError())
self._reset_connection_ds()
def test_one_node_down(self):
"""
If a cass node is down, it tries the next node in cluster
"""
cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace')
self.clients[1].execute.return_value = defer.fail(ConnectionRefusedError())
result = cluster.execute(2, 3)
self.assertEqual(self.successResultOf(result), 'exec_ret3')
self.clients[1].execute.assert_called_once_with(2, 3)
self.clients[2].execute.assert_called_once_with(2, 3)
self.assertFalse(self.clients[0].execute.called)
def test_client_connection_failed(self):
"""
This test is equivalent to txsocksx's
TestSOCKS4ClientEndpoint.test_clientConnectionFailed
"""
args = "host123"
kw = dict()
kw['failure'] = Failure(ConnectionRefusedError())
tor_endpoint = FakeTorSocksEndpoint(*args, **kw)
endpoint = TorClientEndpoint('', 0, socks_endpoint=tor_endpoint)
d = endpoint.connect(None)
return self.assertFailure(d, ConnectionRefusedError)
def test_connectionFailed(self):
"""
The L{Deferred} returned by L{Agent.request} fires with a L{Failure} if
the TCP connection attempt fails.
"""
result = self.agent.request('GET', 'http://foo/')
# Cause the connection to be refused
host, port, factory = self.reactor.tcpClients.pop()[:3]
factory.clientConnectionFailed(None, Failure(ConnectionRefusedError()))
self.completeConnection()
return self.assertFailure(result, ConnectionRefusedError)
def test_bad_port_retry(self):
"""
This tests failure to connect to the ports on the "try" list.
"""
fail_ports = [1984, 666]
for port in fail_ports:
ep = FakeTorSocksEndpoint(
'', '', 0,
accept_port=port,
failure=Failure(ConnectionRefusedError()),
)
endpoint = TorClientEndpoint('', 0, socks_endpoint=ep)
d = endpoint.connect(None)
return self.assertFailure(d, ConnectionRefusedError)
def test_loginConnectionRefused(self):
"""
L{PBClientFactory.login} returns a L{Deferred} which is errbacked
with the L{ConnectionRefusedError} if the underlying connection is
refused.
"""
clientFactory = pb.PBClientFactory()
loginDeferred = clientFactory.login(
credentials.UsernamePassword("foo", "bar"))
clientFactory.clientConnectionFailed(
None,
failure.Failure(
ConnectionRefusedError("Test simulated refused connection")))
return self.assertFailure(loginDeferred, ConnectionRefusedError)
def test_client_connection_failed_user_password(self):
"""
Same as above, but with a username/password.
"""
args = "fakehost"
kw = dict()
kw['failure'] = Failure(ConnectionRefusedError())
tor_endpoint = FakeTorSocksEndpoint(*args, **kw)
endpoint = TorClientEndpoint('invalid host',
0,
socks_username='******',
socks_password='******',
socks_endpoint=tor_endpoint)
d = endpoint.connect(None)
return self.assertFailure(d, ConnectionRefusedError)
def test_all_nodes_down(self):
"""
If all cass nodes are down, it gives up eventually by raising the
connection error exception
"""
cluster = RoundRobinCassandraCluster(['one', 'two', 'three'], 'keyspace')
err = ConnectionRefusedError()
for i in range(3):
self.clients[i].execute.side_effect = lambda *_: defer.fail(err)
result = cluster.execute(2, 3)
self.assertEqual(self.failureResultOf(result).value, err)
for i in range(3):
self.clients[i].execute.assert_called_once_with(2, 3)
def _fakeSubmitRequest(iself, ssl, host, port, request):
if self.refuseConnection:
raise MultiFailure((Failure(ConnectionRefusedError()),))
else:
pod = (port - 8008) / 100
inbox = IScheduleInboxResource(self.site.resource, self.theStoreUnderTest(pod), podding=True)
response = yield inbox.http_POST(SimpleRequest(
self.site,
"POST",
"http://{host}:{port}/podding".format(host=host, port=port),
request.headers,
request.stream.mem,
))
returnValue(response)
def test_client_connection_failed(self, ggt):
"""
This test is equivalent to txsocksx's
TestSOCKS4ClientEndpoint.test_clientConnectionFailed
"""
tor_endpoint = FakeTorSocksEndpoint(
None, "host123", 9050,
failure=Failure(ConnectionRefusedError()),
)
endpoint = TorClientEndpoint(
'', 0,
socks_endpoint=tor_endpoint,
)
d = endpoint.connect(None)
return self.assertFailure(d, ConnectionRefusedError)
def test_client_connection_failed_user_password(self):
"""
Same as above, but with a username/password.
"""
tor_endpoint = FakeTorSocksEndpoint(
None, "fakehose", 9050,
failure=Failure(ConnectionRefusedError()),
)
endpoint = TorClientEndpoint(
'invalid host', 0,
socks_username='******', socks_password='******',
socks_endpoint=tor_endpoint)
d = endpoint.connect(None)
# XXX we haven't fixed socks.py to support user/pw yet ...
return self.assertFailure(d, RuntimeError)
return self.assertFailure(d, ConnectionRefusedError)
def test_connectFailNotify(self, ccf):
"""
Check that if the connection fails to come up that the brokerclient
errback's the deferred returned from the '_connect' call.
"""
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor, 'test_connectFailNotify', 9092,
'clientId')
# attempt connection
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
conn = c.connector
# Claim the connection failed
e = ConnectionRefusedError()
conn.connectionFailed(e)
# Check that the brokerclient called super to reconnect
ccf.assert_called_once_with(c, conn, e)
def test_startServiceReconnectAfterFailure(self):
"""
When the first connection attempt fails, retry.
"""
comp = TestableComponent('example.org', 5347, 'test.example.org',
'secret')
# Starting the service initiates a connection attempt.
comp.startService()
connector = comp._connection
self.assertEqual(1, connector.connects)
# Fail the connection.
connector.connectionFailed(ConnectionRefusedError())
# After a back-off delay, a new connection is attempted.
comp.factory.clock.advance(5)
self.assertEqual(2, connector.connects)
def test_expected_port_probe_failed(self):
"""
If probing for the expected port fails, the probe will be retried.
"""
self.protocol.expectedPort = 1234
self.protocol.makeConnection(self.process)
self.reactor.advance(self.protocol.minUptime)
self.assertThat(self.protocol.ready, has_no_result())
factory = self.reactor.tcpClients[0][2]
factory.clientConnectionFailed(None, ConnectionRefusedError())
self.assertIn("Service port probe failed", self.logger.output)
self.reactor.advance(0.1)
factory = self.reactor.tcpClients[1][2]
factory.buildProtocol(None).connectionMade()
self.assertThat(self.protocol.ready, succeeded(Is(None)))
self.assertIn("Service opened port 1234", self.logger.output)
def test_good_port_retry(self):
"""
This tests that our Tor client endpoint retry logic works correctly.
We create a proxy endpoint that fires a ConnectionRefusedError
unless the connecting port matches. We attempt to connect with the
proxy endpoint for each port that the Tor client endpoint will try.
"""
success_ports = TorClientEndpoint.socks_ports_to_try
for port in success_ports:
tor_endpoint = FakeTorSocksEndpoint(
"fakehost",
"127.0.0.1",
port,
accept_port=port,
failure=Failure(ConnectionRefusedError()),
)
endpoint = TorClientEndpoint('', 0, socks_endpoint=tor_endpoint)
endpoint.connect(None)
self.assertEqual(tor_endpoint.transport.value(), '\x05\x01\x00')
def test_stopServiceNoReconnect(self):
"""
When the service is stopped, no reconnect is attempted.
"""
comp = TestableComponent('example.org', 5347, 'test.example.org',
'secret')
# Starting the service initiates a connection attempt.
comp.startService()
connector = comp._connection
# Fail the connection.
connector.connectionFailed(ConnectionRefusedError())
# If the service is stopped before the back-off delay expires,
# no new connection is attempted.
comp.factory.clock.advance(1)
comp.stopService()
comp.factory.clock.advance(4)
self.assertEqual(1, connector.connects)
def test_connectionFailed(self):
"""
If a connection cannot be established, the L{Deferred} returned by
L{SSHCommandClientEndpoint.connect} fires with a L{Failure}
representing the reason for the connection setup failure.
"""
endpoint = SSHCommandClientEndpoint.newConnection(
self.reactor,
b"/bin/ls -l",
b"dummy user",
self.hostname,
self.port,
knownHosts=self.knownHosts,
ui=FixedResponseUI(False))
factory = Factory()
factory.protocol = Protocol
d = endpoint.connect(factory)
factory = self.reactor.tcpClients[0][2]
factory.clientConnectionFailed(None, Failure(ConnectionRefusedError()))
self.failureResultOf(d).trap(ConnectionRefusedError)
def test_diagnose_wraps_connection_and_api_errors(self):
"""
:func:`diagnose` wraps only :class:`ConnectionRefusedError` and
:class:`APIError`
"""
af = fail(APIError(200, {}))
f = self.failureResultOf(diagnose("system", "operation")(lambda: af)(),
UpstreamError)
self.assertTrue(f.value.reason.check(APIError))
self.assertEqual(f.value.system, "system")
self.assertEqual(f.value.operation, "operation")
cf = fail(ConnectionRefusedError('meh'))
f = self.failureResultOf(diagnose("system", "operation")(lambda: cf)(),
UpstreamError)
self.assertTrue(f.value.reason.check(ConnectionRefusedError))
self.assertEqual(f.value.system, "system")
self.assertEqual(f.value.operation, "operation")
of = fail(ValueError('not-wrapped'))
self.failureResultOf(diagnose("system", "operation")(lambda: of)(),
ValueError)
# Copyright (c) str4d <[email protected]> # See COPYING for details. from twisted.internet.error import ConnectionLost, ConnectionRefusedError from twisted.python import failure from twisted.trial import unittest from txi2p.bob import endpoints from txi2p.test.util import FakeEndpoint, FakeFactory connectionLostFailure = failure.Failure(ConnectionLost()) connectionRefusedFailure = failure.Failure(ConnectionRefusedError()) class BOBI2PClientEndpointTestCase(unittest.TestCase): """ Tests for I2P client Endpoint backed by the BOB API. """ def test_bobConnectionFailed(self): reactor = object() bobEndpoint = FakeEndpoint(failure=connectionRefusedFailure) endpoint = endpoints.BOBI2PClientEndpoint(reactor, bobEndpoint, '') d = endpoint.connect(None) return self.assertFailure(d, ConnectionRefusedError) def TODO_test_destination(self): reactor = object() bobEndpoint = FakeEndpoint() endpoint = endpoints.BOBI2PClientEndpoint(reactor, bobEndpoint, 'foo.i2p') endpoint.connect(None)
def test_delay_adjustment(self):
delay = 0.5
slot_key = 'www.scrapytest.org'
url = 'http://www.scrapytest.org'
ban_url = 'http://ban.me'
self.spider.crawlera_enabled = True
crawler = self._mock_crawler(self.spider, self.settings)
# ignore spider delay by default
self.spider.download_delay = delay
mw = self.mwcls.from_crawler(crawler)
mw.open_spider(self.spider)
self.assertEqual(self.spider.download_delay, 0)
# preserve original delay
self.spider.download_delay = delay
self.spider.crawlera_preserve_delay = True
mw = self.mwcls.from_crawler(crawler)
mw.open_spider(self.spider)
self.assertEqual(self.spider.download_delay, delay)
slot = MockedSlot(self.spider.download_delay)
crawler.engine.downloader.slots[slot_key] = slot
# ban without retry-after
req = Request(url, meta={'download_slot': slot_key})
headers = {'X-Crawlera-Error': 'banned'}
res = self._mock_crawlera_response(
ban_url,
status=self.bancode,
headers=headers,
)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
# ban with retry-after
retry_after = 1.5
headers = {
'retry-after': str(retry_after),
'X-Crawlera-Error': 'banned'
}
res = self._mock_crawlera_response(
ban_url,
status=self.bancode,
headers=headers,
)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, retry_after)
self.assertEqual(self.spider.download_delay, delay)
# DNS cache should be cleared in case of errors
dnscache['proxy.crawlera.com'] = '1.1.1.1'
res = self._mock_crawlera_response(url)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
self.assertIn('proxy.crawlera.com', dnscache)
# server failures
mw.process_exception(req, ConnectionRefusedError(), self.spider)
self.assertEqual(slot.delay, mw.connection_refused_delay)
self.assertEqual(self.spider.download_delay, delay)
self.assertNotIn('proxy.crawlera.com', dnscache)
dnscache['proxy.crawlera.com'] = '1.1.1.1'
res = self._mock_crawlera_response(ban_url)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
self.assertIn('proxy.crawlera.com', dnscache)
mw.process_exception(req, ConnectionRefusedError(), self.spider)
self.assertEqual(slot.delay, mw.connection_refused_delay)
self.assertEqual(self.spider.download_delay, delay)
self.assertNotIn('proxy.crawlera.com', dnscache)
dnscache['proxy.crawlera.com'] = '1.1.1.1'
res = self._mock_crawlera_response(ban_url, status=self.bancode)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
self.assertIn('proxy.crawlera.com', dnscache)
mw.process_exception(req, ConnectionDone(), self.spider)
self.assertEqual(slot.delay, mw.connection_refused_delay)
self.assertEqual(self.spider.download_delay, delay)
self.assertNotIn('proxy.crawlera.com', dnscache)
def test_delay_reset(self):
"""
Test that reconnect delay is handled correctly:
1) That initializer values are respected
2) That delay maximum is respected
3) That delay is reset to initial delay on successful connection
"""
init_delay = last_delay = 0.025
max_delay = 14
reactor = MemoryReactorClock()
c = KafkaBrokerClient(reactor,
'test_delay_reset',
9092,
'clientId',
initDelay=init_delay,
maxDelay=max_delay)
c.jitter = 0 # Eliminate randomness for test
# Ensure KBC was initialized correctly
self.assertEqual(c.retries, 0)
self.assertEqual(c.delay, init_delay)
self.assertEqual(c.maxDelay, max_delay)
self.assertTrue(c.continueTrying)
c._connect() # Force a connection attempt
c.connector.factory = c # MemoryReactor doesn't make this connection.
self.assertTrue(c.connector.connectCalled)
# Reset it so we can track future calls
c.connector.connectCalled = False
# Build the protocol, like a real connector would on successful connect
c.buildProtocol(None)
# Fake server connection close
f = Failure(ConnectionDone('test_delay_reset'))
c.clientConnectionLost(c.connector, f)
# Now loop failing connection attempts until we get to the max
while c.delay < max_delay:
# Assert a reconnect wasn't immediately attempted
self.assertFalse(c.connector.connectCalled)
# Assert the new delay was calculated correctly
self.assertEqual(last_delay * c.factor, c.delay)
last_delay = c.delay
# advance the reactor, but not enough to connect
reactor.advance(0.1 * c.delay)
# Still no connection
self.assertFalse(c.connector.connectCalled)
# Should see a connection attempt after this
reactor.advance(c.delay)
self.assertTrue(c.connector.connectCalled)
c.connector.connectCalled = False # Reset again
# Claim the connection failed
e = ConnectionRefusedError()
c.connector.connectionFailed(e)
# Assert the delay was calculated correctly
self.assertEqual(max_delay, c.delay)
self.assertFalse(c.connector.connectCalled)
# "run" the reactor, but not enough to connect
reactor.advance(0.1 * c.delay)
# Still no connection
self.assertFalse(c.connector.connectCalled)
# Should see a connection attempt after this
reactor.advance(c.delay)
self.assertTrue(c.connector.connectCalled)
# Build the protocol, like a real connector would on successful connect
c.buildProtocol(None)
# Assert that the delay and retry count were reset
self.assertEqual(init_delay, c.delay)
self.assertEqual(c.retries, 0)
def test_delay_adjustment(self):
delay = 0.5
slot_key = 'www.scrapytest.org'
url = 'http://www.scrapytest.org'
ban_url = 'http://ban.me'
self.spider.crawlera_enabled = True
crawler = self._mock_crawler(self.settings)
# ignore spider delay by default
self.spider.download_delay = delay
mw = self.mwcls.from_crawler(crawler)
mw.open_spider(self.spider)
self.assertEqual(self.spider.download_delay, 0)
# preserve original delay
self.spider.download_delay = delay
self.spider.crawlera_preserve_delay = True
mw = self.mwcls.from_crawler(crawler)
mw.open_spider(self.spider)
self.assertEqual(self.spider.download_delay, delay)
slot = MockedSlot(self.spider.download_delay)
crawler.engine.downloader.slots[slot_key] = slot
# ban
req = Request(url, meta={'download_slot': slot_key})
res = Response(ban_url, status=self.bancode, request=req)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
retry_after = 1.5
headers = {'retry-after': str(retry_after)}
res = Response(ban_url,
status=self.bancode,
headers=headers,
request=req)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, retry_after)
self.assertEqual(self.spider.download_delay, delay)
res = Response(url, request=req)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
# server failures
mw.process_exception(req, ConnectionRefusedError(), self.spider)
self.assertEqual(slot.delay, mw.connection_refused_delay)
self.assertEqual(self.spider.download_delay, delay)
res = Response(ban_url, request=req)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
mw.process_exception(req, ConnectionRefusedError(), self.spider)
self.assertEqual(slot.delay, mw.connection_refused_delay)
self.assertEqual(self.spider.download_delay, delay)
res = Response(ban_url, status=self.bancode, request=req)
mw.process_response(req, res, self.spider)
self.assertEqual(slot.delay, delay)
self.assertEqual(self.spider.download_delay, delay)
def test_connection_error(self):
failure = Failure(ConnectionRefusedError("timeout"))
error = self.assertRaises(Exception, error_wrapper, failure,
ConnectionRefusedError)
self.assertTrue(isinstance(error, ConnectionRefusedError))
