def test_connection(self, connect_mock): mock_sock = FakeSocket() host, port = ('foobar', 9000) connect_timeout, recv_timeout = (1, 1) connect_mock.return_value = mock_sock pool = TCPConnectionPool(host, port, 1, connect_timeout, recv_timeout) with pool.connection() as sock: assert sock == mock_sock assert mock_sock.call_args_list == [('settimeout', mock.call(recv_timeout))] assert connect_mock.call_args_list == [ mock.call((host, port), timeout=connect_timeout) ] # After using a connection, it should be checked-in/cached for others to use connect_mock.return_value = FakeSocket() assert not pool.pool.empty() with pool.connection() as sock: assert sock == mock_sock # When we are at capacity, then callers will block # waiting for a connection to be returned. with pytest.raises(EmptyPoolException): with pool.connection(_block=False): assert False # Should not be reached
def test_closing_tolerates_close_exceptions(self, create_mock): expected_sock = FakeSocket(close_failures=10) create_mock.return_value = expected_sock pool = TCPConnectionPool('foobar', 80, 10, 1, 1) with pool.connection(): pass pool.closeall() assert expected_sock.call_args_list[-1] == ('close', mock.call())
def test_defunct_connections_are_regenerated(self, connect_mock): pool = TCPConnectionPool('foobar', 80, 10, 1, 1) mock_sock = FakeSocket() connect_mock.return_value = mock_sock pool.pool = Queue() pool.pool.put(_DefunctConnection) with pool.connection() as conn: assert conn is mock_sock
def test_create_connection_reraises(self, create_connection_mock): pool = TCPConnectionPool('foobar', 80, 10, 1, 1) with pytest.raises(ForcedException): with pool.connection(): assert False # Should not be reached. # No connections become cached. assert pool.pool.qsize() == 0 assert pool.size == 0
def test_exceptions_result_in_defunct_connections(self, connect_mock): size = 10 pool = TCPConnectionPool('foobar', 80, size, 1, 1) mock_sock = FakeSocket() connect_mock.return_value = mock_sock with pytest.raises(ForcedException): with pool.connection(): raise ForcedException() assert pool.pool.qsize() == size assert pool.pool.get_nowait() == _DefunctConnection
def __init__(self, host=None, port=None, connect_timeout=50, publish_timeout=10, maxsize=10): host = host or pycernan.avro.config.host() port = port or pycernan.avro.config.port() self.connect_timeout = connect_timeout self.publish_timeout = publish_timeout self.pool = TCPConnectionPool(host, port, maxsize=maxsize, connect_timeout=connect_timeout, read_timeout=publish_timeout)
def test_pool_creation(self, connection_mock): size = 10 pool = TCPConnectionPool('foobar', 80, size, 3, 1) assert pool.pool is not None for _ in range(size): assert pool.pool.get_nowait() == _DefunctConnection # Pool should be drained now. with pytest.raises(Empty): pool.pool.get_nowait()
def test_connection_on_exception_closes_connections(self, connect_mock): pool = TCPConnectionPool('foobar', 8080, 1, 1, 1) # Connections are closed when application layer Exceptions occur. mock_sock = FakeSocket() connect_mock.return_value = mock_sock with pytest.raises(ForcedException): with pool.connection() as sock: raise ForcedException() assert mock_sock.call_args_list[-1] == ('close', mock.call()) # Repeat the same experiment, this time simulating exception on close. # The semantics should match, namely, the exception is tolerated and # users get back the application level Exception. mock_sock.close_failures = 10 mock_sock.call_args_list = [] with pytest.raises(ForcedException): with pool.connection() as sock: raise ForcedException() assert mock_sock.call_args_list[-1] == ('close', mock.call())
def test_regenerating_connections_tolerates_exceptions(self, connect_mock): num_failures = 20 pool = TCPConnectionPool('foobar', 80, 10, 1, 1) mock_sock = FakeSocket(set_failures=num_failures) connect_mock.return_value = mock_sock # Simulate a prior failure pool.pool = Queue() pool.size = 1 pool.pool._put(_DefunctConnection) # Try a number of failed operations. # Each time the defunct connection should be returned to the queue # for subsequent calls to reattempt. for i in range(num_failures): with pytest.raises(Exception): with pool.connection(): assert False # Should not be reached assert pool.pool.qsize() == pool.size == 1 assert pool.pool.get_nowait() is _DefunctConnection pool.pool._put(_DefunctConnection) # Failures are exhausted, we should be able to now # regenerate a valid context. with pool.connection(): pass assert pool.pool.qsize() == 1 assert pool.pool.get_nowait() is mock_sock
def test_pool_value_errors(self): with pytest.raises(ValueError): TCPConnectionPool('foobar', 80, -1, 1, 1)
class Client(object): """ Interface specification for all Avro clients. """ __metaclass__ = ABCMeta def __init__(self, host=None, port=None, connect_timeout=50, publish_timeout=10, maxsize=10): host = host or pycernan.avro.config.host() port = port or pycernan.avro.config.port() self.connect_timeout = connect_timeout self.publish_timeout = publish_timeout self.pool = TCPConnectionPool(host, port, maxsize=maxsize, connect_timeout=connect_timeout, read_timeout=publish_timeout) def close(self): """ Closes all previously established connections not actively in use. """ self.pool.closeall() @metrics.publish_failure_count.count_exceptions() def publish(self, schema_map, batch, ephemeral_storage=False, **kwargs): """ Publishes a batch of records corresponding to the given schema. Args: schema_map: dict - Avro schema defintion. batch: list - List of Avro records (as dicts). Kwargs: ephemeral_storage: bool - Flag to indicate whether the batch should be stored long-term. Others are version specific options. See extending object. """ if not batch: raise EmptyBatchException() blob = serialize(schema_map, batch, ephemeral_storage) self.publish_blob(blob, **kwargs) def publish_file(self, file_path, **kwargs): """ Reads and publishes an Avro encoded file. Args: file_path : string - Path to the file. Kwargs: Version specific options. See extending object. """ with open(file_path, "rb") as file: avro_blob = file.read() self.publish_blob(avro_blob, **kwargs) @abstractmethod def publish_blob(self, avro_blob, **kwargs): """ Version specific payload generation / publication. """ pass # pragma: no cover