def _test(self, factory, testvalue):
transport = StringTransport()
protocol = client.ScrapyHTTPPageGetter()
protocol.factory = factory
protocol.makeConnection(transport)
self.assertEqual(set(transport.value().splitlines()),
set(testvalue.splitlines()))
return testvalue
def setUp(self) -> None:
self.factory = ServerProtocolFactory()
self.proto = self.factory.buildProtocol(('127.0.0.1', 6379))
self.tr = StringTransport()
self.proto.makeConnection(self.tr)
self.output = ''
def fake_print(s):
self.output = s
self.fake_print = fake_print
def test_data_cancels_timeout(self):
"""
When data is received, the timeout is canceled
"""
clock = Clock()
protocol = HangCheckProtocol(Protocol(), reactor=clock)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
protocol.dataReceived(b'some-data')
assert_clock_idle(self, clock)
def test_transport(self):
"""
The transport passed to the underlying protocol is
the underlying transport.
"""
clock = Clock()
wrapped_protocol = Protocol()
protocol = HangCheckProtocol(wrapped_protocol, reactor=clock)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
self.assertIdentical(wrapped_protocol.transport, transport)
def test_forwards_data(self):
"""
Data received by the protocol gets passed to the wrapped
protocol.
"""
clock = Clock()
wrapped_protocol = AccumulatingProtocol()
protocol = HangCheckProtocol(wrapped_protocol, reactor=clock)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
protocol.dataReceived(b'some-data')
self.assertEqual(wrapped_protocol.data, b"some-data")
def test_disconnects(self):
"""
When connecting to a server that doesn't send any data,
the protocol disconnects after the timeout.
"""
clock = Clock()
protocol = HangCheckProtocol(Protocol(), reactor=clock)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
clock.advance(120)
self.assertTrue(transport.disconnecting)
assert_clock_idle(self, clock)
def test_defaultPeerAddress(self):
"""
If no peer address is passed to L{StringTransport.__init__}, an
L{IPv4Address} is returned from L{StringTransport.getPeer}.
"""
address = StringTransport().getPeer()
self.assertIsInstance(address, IPv4Address)
def test_specifiedPeerAddress(self):
"""
If a peer address is passed to L{StringTransport.__init__}, that
value is returned from L{StringTransport.getPeer}.
"""
address = object()
self.assertIs(StringTransport(peerAddress=address).getPeer(), address)
def test_specifiedHostAddress(self):
"""
If a host address is passed to L{StringTransport.__init__}, that
value is returned from L{StringTransport.getHost}.
"""
address = object()
self.assertIs(StringTransport(address).getHost(), address)
def setUp(self) -> None:
self.factory = ClientProtocolFactory()
self.addr = IPv4Address('TCP', '127.0.0.1', 50000)
self.tr = StringTransport()
self.input = ''
self.output = ''
def fake_input(s):
return self.input
def fake_print(s):
self.output = s
self.fake_input = fake_input
self.fake_print = fake_print
def setUp(self):
self.factory = WebSocketServerFactory()
self.factory.setProtocolOptions(trustXForwardedFor=2)
self.proto = WebSocketServerProtocol()
self.proto.transport = StringTransport()
self.proto.factory = self.factory
self.proto.failHandshake = Mock()
self.proto._connectionMade()
def test_overlong_requests_are_rejected(self):
# as a control case, first send a regular request.
# connect the site to a fake transport.
transport = StringTransport()
protocol = self.site.buildProtocol(IPv6Address("TCP", "::1", 2345))
protocol.makeConnection(transport)
protocol.dataReceived(
b"POST / HTTP/1.1\r\n"
b"Connection: close\r\n"
b"Transfer-Encoding: chunked\r\n"
b"\r\n"
b"0\r\n"
b"\r\n"
)
# we should get a 404
self.assertRegex(transport.value().decode(), r"^HTTP/1\.1 404 ")
# now send an oversized request
transport = StringTransport()
protocol = self.site.buildProtocol(IPv6Address("TCP", "::1", 2345))
protocol.makeConnection(transport)
protocol.dataReceived(
b"POST / HTTP/1.1\r\n"
b"Connection: close\r\n"
b"Transfer-Encoding: chunked\r\n"
b"\r\n"
)
# we deliberately send all the data in one big chunk, to ensure that
# twisted isn't buffering the data in the chunked transfer decoder.
# we start with the chunk size, in hex. (We won't actually send this much)
protocol.dataReceived(b"10000000\r\n")
sent = 0
while not transport.disconnected:
self.assertLess(sent, 0x10000000, "connection did not drop")
protocol.dataReceived(b"\0" * 1024)
sent += 1024
# default max upload size is 512K, so it should drop on the next buffer after
# that.
self.assertEqual(sent, 513 * 1024)
def setUp(self):
self.factory = WebSocketServerFactory()
self.factory.setProtocolOptions(
allowedOrigins=[u'127.0.0.1:*', u'*.example.com:*'])
self.proto = WebSocketServerProtocol()
self.proto.transport = StringTransport()
self.proto.factory = self.factory
self.proto.failHandshake = Mock()
self.proto._connectionMade()
def test_disconnect_cancels_timeout(self):
"""
If the connection is closed, the hang check is cancelled.
"""
clock = Clock()
protocol = HangCheckProtocol(
Protocol(),
reactor=clock,
)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
protocol.connectionLost(
Failure(ConnectionDone("Bye."))
)
assert_clock_idle(self, clock)
def test_data_and_disconnect(self):
"""
If the connection receives data and then is closed, no error results.
"""
clock = Clock()
protocol = HangCheckProtocol(
Protocol(),
reactor=clock,
)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
protocol.dataReceived(b"some-data")
protocol.connectionLost(
Failure(ConnectionDone("Bye."))
)
assert_clock_idle(self, clock)
def test_calls_callback(self):
"""
When connecting to a server that doesn't send any data,
the protocol calls the hung callback.
"""
results = []
clock = Clock()
protocol = HangCheckProtocol(
Protocol(),
hung_callback=lambda: results.append(True),
reactor=clock,
)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
clock.advance(120)
self.assertEqual(results, [True])
assert_clock_idle(self, clock)
def test_disconnect_forwarded(self):
"""
If the connection is closed, the underlying protocol is informed.
"""
clock = Clock()
wrapped_protocol = AccumulatingProtocol()
protocol = HangCheckProtocol(wrapped_protocol, reactor=clock)
transport = StringTransport()
transport.protocol = protocol
protocol.makeConnection(transport)
reason = ConnectionDone("Bye.")
protocol.connectionLost(
Failure(reason)
)
self.assertTrue(wrapped_protocol.closed)
self.assertEqual(
wrapped_protocol.closedReason.value,
reason,
)
class TestServerProtocol(unittest.TestCase):
def setUp(self) -> None:
self.factory = ClientProtocolFactory()
self.addr = IPv4Address('TCP', '127.0.0.1', 50000)
self.tr = StringTransport()
self.input = ''
self.output = ''
def fake_input(s):
return self.input
def fake_print(s):
self.output = s
self.fake_input = fake_input
self.fake_print = fake_print
def test_send_receive(self):
"""
Test sending and receiving data
"""
with patch('builtins.input', self.fake_input):
with patch('builtins.print', self.fake_print):
self.proto = self.factory.buildProtocol(self.addr)
self.input = 'set 1 one'
self.proto.makeConnection(self.tr)
command = RedisEncoder.encodeArray(['set', '1', 'one'])
self.assertEqual(command, self.tr.value())
self.tr.clear()
self.input = 'get 1'
self.proto.dataReceived(b'+OK\r\n')
self.assertEqual('OK', self.output)
command = RedisEncoder.encodeArray(['get', '1'])
self.assertEqual(command, self.tr.value())
def test_exit(self):
"""
Test disconnecting on 'exit' prompt
"""
with patch('builtins.input', self.fake_input):
with patch('builtins.print', self.fake_print):
self.proto = self.factory.buildProtocol(self.addr)
self.input = 'exit'
self.proto.makeConnection(self.tr)
self.assertEqual(True, self.tr.disconnecting)
def test_disconnect_on_garbage(self):
"""
Test disconnecting when garbage received (no first bit in data)
"""
with patch('builtins.input', self.fake_input):
with patch('builtins.print', self.fake_print):
self.proto = self.factory.buildProtocol(self.addr)
self.input = 'set 1 one'
self.proto.makeConnection(self.tr)
command = RedisEncoder.encodeArray(['set', '1', 'one'])
self.assertEqual(command, self.tr.value())
self.tr.clear()
self.proto.dataReceived(b'OK\r\n')
self.assertEqual(True, self.tr.disconnecting)
class StringTransportTests(TestCase):
"""
Tests for L{twisted.internet.testing.StringTransport}.
"""
def setUp(self):
self.transport = StringTransport()
def test_interfaces(self):
"""
L{StringTransport} instances provide L{ITransport}, L{IPushProducer},
and L{IConsumer}.
"""
self.assertTrue(verifyObject(ITransport, self.transport))
self.assertTrue(verifyObject(IPushProducer, self.transport))
self.assertTrue(verifyObject(IConsumer, self.transport))
def test_registerProducer(self):
"""
L{StringTransport.registerProducer} records the arguments supplied to
it as instance attributes.
"""
producer = object()
streaming = object()
self.transport.registerProducer(producer, streaming)
self.assertIs(self.transport.producer, producer)
self.assertIs(self.transport.streaming, streaming)
def test_disallowedRegisterProducer(self):
"""
L{StringTransport.registerProducer} raises L{RuntimeError} if a
producer is already registered.
"""
producer = object()
self.transport.registerProducer(producer, True)
self.assertRaises(RuntimeError, self.transport.registerProducer,
object(), False)
self.assertIs(self.transport.producer, producer)
self.assertTrue(self.transport.streaming)
def test_unregisterProducer(self):
"""
L{StringTransport.unregisterProducer} causes the transport to forget
about the registered producer and makes it possible to register a new
one.
"""
oldProducer = object()
newProducer = object()
self.transport.registerProducer(oldProducer, False)
self.transport.unregisterProducer()
self.assertIsNone(self.transport.producer)
self.transport.registerProducer(newProducer, True)
self.assertIs(self.transport.producer, newProducer)
self.assertTrue(self.transport.streaming)
def test_invalidUnregisterProducer(self):
"""
L{StringTransport.unregisterProducer} raises L{RuntimeError} if called
when no producer is registered.
"""
self.assertRaises(RuntimeError, self.transport.unregisterProducer)
def test_initialProducerState(self):
"""
L{StringTransport.producerState} is initially C{'producing'}.
"""
self.assertEqual(self.transport.producerState, "producing")
def test_pauseProducing(self):
"""
L{StringTransport.pauseProducing} changes the C{producerState} of the
transport to C{'paused'}.
"""
self.transport.pauseProducing()
self.assertEqual(self.transport.producerState, "paused")
def test_resumeProducing(self):
"""
L{StringTransport.resumeProducing} changes the C{producerState} of the
transport to C{'producing'}.
"""
self.transport.pauseProducing()
self.transport.resumeProducing()
self.assertEqual(self.transport.producerState, "producing")
def test_stopProducing(self):
"""
L{StringTransport.stopProducing} changes the C{'producerState'} of the
transport to C{'stopped'}.
"""
self.transport.stopProducing()
self.assertEqual(self.transport.producerState, "stopped")
def test_stoppedTransportCannotPause(self):
"""
L{StringTransport.pauseProducing} raises L{RuntimeError} if the
transport has been stopped.
"""
self.transport.stopProducing()
self.assertRaises(RuntimeError, self.transport.pauseProducing)
def test_stoppedTransportCannotResume(self):
"""
L{StringTransport.resumeProducing} raises L{RuntimeError} if the
transport has been stopped.
"""
self.transport.stopProducing()
self.assertRaises(RuntimeError, self.transport.resumeProducing)
def test_disconnectingTransportCannotPause(self):
"""
L{StringTransport.pauseProducing} raises L{RuntimeError} if the
transport is being disconnected.
"""
self.transport.loseConnection()
self.assertRaises(RuntimeError, self.transport.pauseProducing)
def test_disconnectingTransportCannotResume(self):
"""
L{StringTransport.resumeProducing} raises L{RuntimeError} if the
transport is being disconnected.
"""
self.transport.loseConnection()
self.assertRaises(RuntimeError, self.transport.resumeProducing)
def test_loseConnectionSetsDisconnecting(self):
"""
L{StringTransport.loseConnection} toggles the C{disconnecting} instance
variable to C{True}.
"""
self.assertFalse(self.transport.disconnecting)
self.transport.loseConnection()
self.assertTrue(self.transport.disconnecting)
def test_specifiedHostAddress(self):
"""
If a host address is passed to L{StringTransport.__init__}, that
value is returned from L{StringTransport.getHost}.
"""
address = object()
self.assertIs(StringTransport(address).getHost(), address)
def test_specifiedPeerAddress(self):
"""
If a peer address is passed to L{StringTransport.__init__}, that
value is returned from L{StringTransport.getPeer}.
"""
address = object()
self.assertIs(StringTransport(peerAddress=address).getPeer(), address)
def test_defaultHostAddress(self):
"""
If no host address is passed to L{StringTransport.__init__}, an
L{IPv4Address} is returned from L{StringTransport.getHost}.
"""
address = StringTransport().getHost()
self.assertIsInstance(address, IPv4Address)
def test_defaultPeerAddress(self):
"""
If no peer address is passed to L{StringTransport.__init__}, an
L{IPv4Address} is returned from L{StringTransport.getPeer}.
"""
address = StringTransport().getPeer()
self.assertIsInstance(address, IPv4Address)
def setUp(self):
self.transport = StringTransport()
class TestServerProtocol(unittest.TestCase):
def setUp(self) -> None:
self.factory = ServerProtocolFactory()
self.proto = self.factory.buildProtocol(('127.0.0.1', 6379))
self.tr = StringTransport()
self.proto.makeConnection(self.tr)
self.output = ''
def fake_print(s):
self.output = s
self.fake_print = fake_print
def test_connection_lost(self):
factory = ServerProtocolFactory()
proto = factory.buildProtocol(('127.0.0.1', 6379))
tr = StringTransportWithDisconnection()
tr.protocol = proto
proto.makeConnection(self.tr)
tr.loseConnection()
self.assertEqual(0, factory.proto_count)
def test_set_get(self):
"""
Set and then get key
:return:
"""
data = RedisEncoder.encodeArray(['set', '1', 'one'])
self.proto.dataReceived(data)
self.assertEqual(b'+OK\r\n', self.tr.value())
self.tr.clear()
data = RedisEncoder.encodeArray(['get', '1'])
self.proto.dataReceived(data)
self.assertEqual(b'$3\r\none\r\n', self.tr.value())
self.tr.clear()
def test_get_None(self):
"""
Get key that is not set
:return:
"""
data = RedisEncoder.encodeArray(['get', 'nonset'])
self.proto.dataReceived(data)
self.assertEqual(b'$-1\r\n', self.tr.value())
self.tr.clear()
def test_lrange_None(self):
"""
Get array that is not set
:return:
"""
data = RedisEncoder.encodeArray(['lrange', 'nonset', '0', '1'])
self.proto.dataReceived(data)
self.assertEqual(b'*-1\r\n', self.tr.value())
self.tr.clear()
def test_wrong_first_byte(self):
"""
Receive data with wrong first byte,
then receive normal data.
:return:
"""
with patch('builtins.print', self.fake_print):
data = b'abc'
self.proto.dataReceived(data)
self.assertEqual(b'', self.tr.value())
self.assertEqual(b'', self.proto._data_buffer)
data = RedisEncoder.encodeArray(['set', '1', 'one'])
self.proto.dataReceived(data)
self.assertEqual(b'+OK\r\n', self.tr.value())
self.tr.clear()