def test_connection(self, connect_mock):
mock_sock = FakeSocket()
host, port = ('foobar', 9000)
connect_timeout, recv_timeout = (1, 1)
connect_mock.return_value = mock_sock
pool = TCPConnectionPool(host, port, 1, connect_timeout, recv_timeout)
with pool.connection() as sock:
assert sock == mock_sock
assert mock_sock.call_args_list == [('settimeout',
mock.call(recv_timeout))]
assert connect_mock.call_args_list == [
mock.call((host, port), timeout=connect_timeout)
]
# After using a connection, it should be checked-in/cached for others to use
connect_mock.return_value = FakeSocket()
assert not pool.pool.empty()
with pool.connection() as sock:
assert sock == mock_sock
# When we are at capacity, then callers will block
# waiting for a connection to be returned.
with pytest.raises(EmptyPoolException):
with pool.connection(_block=False):
assert False # Should not be reached
def test_closing_tolerates_close_exceptions(self, create_mock):
expected_sock = FakeSocket(close_failures=10)
create_mock.return_value = expected_sock
pool = TCPConnectionPool('foobar', 80, 10, 1, 1)
with pool.connection():
pass
pool.closeall()
assert expected_sock.call_args_list[-1] == ('close', mock.call())
def test_defunct_connections_are_regenerated(self, connect_mock):
pool = TCPConnectionPool('foobar', 80, 10, 1, 1)
mock_sock = FakeSocket()
connect_mock.return_value = mock_sock
pool.pool = Queue()
pool.pool.put(_DefunctConnection)
with pool.connection() as conn:
assert conn is mock_sock
def test_create_connection_reraises(self, create_connection_mock):
pool = TCPConnectionPool('foobar', 80, 10, 1, 1)
with pytest.raises(ForcedException):
with pool.connection():
assert False # Should not be reached.
# No connections become cached.
assert pool.pool.qsize() == 0
assert pool.size == 0
def test_exceptions_result_in_defunct_connections(self, connect_mock):
size = 10
pool = TCPConnectionPool('foobar', 80, size, 1, 1)
mock_sock = FakeSocket()
connect_mock.return_value = mock_sock
with pytest.raises(ForcedException):
with pool.connection():
raise ForcedException()
assert pool.pool.qsize() == size
assert pool.pool.get_nowait() == _DefunctConnection
def __init__(self,
host=None,
port=None,
connect_timeout=50,
publish_timeout=10,
maxsize=10):
host = host or pycernan.avro.config.host()
port = port or pycernan.avro.config.port()
self.connect_timeout = connect_timeout
self.publish_timeout = publish_timeout
self.pool = TCPConnectionPool(host,
port,
maxsize=maxsize,
connect_timeout=connect_timeout,
read_timeout=publish_timeout)
def test_pool_creation(self, connection_mock):
size = 10
pool = TCPConnectionPool('foobar', 80, size, 3, 1)
assert pool.pool is not None
for _ in range(size):
assert pool.pool.get_nowait() == _DefunctConnection
# Pool should be drained now.
with pytest.raises(Empty):
pool.pool.get_nowait()
def test_connection_on_exception_closes_connections(self, connect_mock):
pool = TCPConnectionPool('foobar', 8080, 1, 1, 1)
# Connections are closed when application layer Exceptions occur.
mock_sock = FakeSocket()
connect_mock.return_value = mock_sock
with pytest.raises(ForcedException):
with pool.connection() as sock:
raise ForcedException()
assert mock_sock.call_args_list[-1] == ('close', mock.call())
# Repeat the same experiment, this time simulating exception on close.
# The semantics should match, namely, the exception is tolerated and
# users get back the application level Exception.
mock_sock.close_failures = 10
mock_sock.call_args_list = []
with pytest.raises(ForcedException):
with pool.connection() as sock:
raise ForcedException()
assert mock_sock.call_args_list[-1] == ('close', mock.call())
def test_regenerating_connections_tolerates_exceptions(self, connect_mock):
num_failures = 20
pool = TCPConnectionPool('foobar', 80, 10, 1, 1)
mock_sock = FakeSocket(set_failures=num_failures)
connect_mock.return_value = mock_sock
# Simulate a prior failure
pool.pool = Queue()
pool.size = 1
pool.pool._put(_DefunctConnection)
# Try a number of failed operations.
# Each time the defunct connection should be returned to the queue
# for subsequent calls to reattempt.
for i in range(num_failures):
with pytest.raises(Exception):
with pool.connection():
assert False # Should not be reached
assert pool.pool.qsize() == pool.size == 1
assert pool.pool.get_nowait() is _DefunctConnection
pool.pool._put(_DefunctConnection)
# Failures are exhausted, we should be able to now
# regenerate a valid context.
with pool.connection():
pass
assert pool.pool.qsize() == 1
assert pool.pool.get_nowait() is mock_sock
def test_pool_value_errors(self):
with pytest.raises(ValueError):
TCPConnectionPool('foobar', 80, -1, 1, 1)
class Client(object):
"""
Interface specification for all Avro clients.
"""
__metaclass__ = ABCMeta
def __init__(self,
host=None,
port=None,
connect_timeout=50,
publish_timeout=10,
maxsize=10):
host = host or pycernan.avro.config.host()
port = port or pycernan.avro.config.port()
self.connect_timeout = connect_timeout
self.publish_timeout = publish_timeout
self.pool = TCPConnectionPool(host,
port,
maxsize=maxsize,
connect_timeout=connect_timeout,
read_timeout=publish_timeout)
def close(self):
"""
Closes all previously established connections not actively in use.
"""
self.pool.closeall()
@metrics.publish_failure_count.count_exceptions()
def publish(self, schema_map, batch, ephemeral_storage=False, **kwargs):
"""
Publishes a batch of records corresponding to the given schema.
Args:
schema_map: dict - Avro schema defintion.
batch: list - List of Avro records (as dicts).
Kwargs:
ephemeral_storage: bool - Flag to indicate whether the batch
should be stored long-term.
Others are version specific options. See extending object.
"""
if not batch:
raise EmptyBatchException()
blob = serialize(schema_map, batch, ephemeral_storage)
self.publish_blob(blob, **kwargs)
def publish_file(self, file_path, **kwargs):
"""
Reads and publishes an Avro encoded file.
Args:
file_path : string - Path to the file.
Kwargs:
Version specific options. See extending object.
"""
with open(file_path, "rb") as file:
avro_blob = file.read()
self.publish_blob(avro_blob, **kwargs)
@abstractmethod
def publish_blob(self, avro_blob, **kwargs):
"""
Version specific payload generation / publication.
"""
pass # pragma: no cover