def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < Version("2.1"):
return
start_and_prime_singledc()
cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION, compression=False)
cls.session = cls.cluster.connect(wait_for_all_pools=True)
def test_host_is_not_set_to_down_after_query_oto(self):
"""
Test to ensure that the connections aren't closed if there's an
OperationTimedOut in a normal query. This should only happen from the
heart beat thread (in the case of a OperationTimedOut) with the default
configuration
@since 3.12
@expected_result the connections aren't closed nor the hosts are
set to down
@test_category connection
"""
start_and_prime_singledc()
query_to_prime = "SELECT * FROM madeup_keyspace.madeup_table"
prime_query(query_to_prime, then=NO_THEN)
listener = TrackDownListener()
cluster = Cluster(compression=False)
session = cluster.connect(wait_for_all_pools=True)
cluster.register_listener(listener)
futures = []
for _ in range(10):
future = session.execute_async(query_to_prime)
futures.append(future)
for f in futures:
f._event.wait()
self.assertIsInstance(f._final_exception, OperationTimedOut)
self.assertEqual(listener.hosts_marked_down, [])
assert_quiescent_pool_state(self, cluster)
def test_driver_recovers_nework_isolation(self):
start_and_prime_singledc()
idle_heartbeat_timeout = 3
idle_heartbeat_interval = 1
listener = TrackDownListener()
cluster = Cluster(['127.0.0.1'],
load_balancing_policy=RoundRobinPolicy(),
idle_heartbeat_timeout=idle_heartbeat_timeout,
idle_heartbeat_interval=idle_heartbeat_interval,
executor_threads=16)
session = cluster.connect(wait_for_all_pools=True)
cluster.register_listener(listener)
prime_request(PrimeOptions(then=NO_THEN))
prime_request(RejectConnections(RejectType.REJECT_STARTUP))
time.sleep((idle_heartbeat_timeout + idle_heartbeat_interval) * 2)
for host in cluster.metadata.all_hosts():
self.assertIn(host, listener.hosts_marked_down)
self.assertRaises(NoHostAvailable, session.execute,
"SELECT * from system.local")
clear_queries()
prime_request(AcceptConnections())
time.sleep(idle_heartbeat_timeout + idle_heartbeat_interval + 2)
self.assertIsNotNone(session.execute("SELECT * from system.local"))
def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < "2.1":
return
start_and_prime_singledc()
cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION,
compression=False)
cls.session = cls.cluster.connect(wait_for_all_pools=True)
spec_ep_brr = ExecutionProfile(
load_balancing_policy=BadRoundRobinPolicy(),
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(
1, 6),
request_timeout=12)
spec_ep_rr = ExecutionProfile(
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(
.5, 10),
request_timeout=12)
spec_ep_rr_lim = ExecutionProfile(
load_balancing_policy=BadRoundRobinPolicy(),
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(
0.5, 1),
request_timeout=12)
spec_ep_brr_lim = ExecutionProfile(
load_balancing_policy=BadRoundRobinPolicy(),
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(
4, 10))
cls.cluster.add_execution_profile("spec_ep_brr", spec_ep_brr)
cls.cluster.add_execution_profile("spec_ep_rr", spec_ep_rr)
cls.cluster.add_execution_profile("spec_ep_rr_lim", spec_ep_rr_lim)
cls.cluster.add_execution_profile("spec_ep_brr_lim", spec_ep_brr_lim)
def test_driver_recovers_nework_isolation(self):
start_and_prime_singledc()
idle_heartbeat_timeout = 3
idle_heartbeat_interval = 1
listener = TrackDownListener()
cluster = Cluster(['127.0.0.1'],
load_balancing_policy=RoundRobinPolicy(),
idle_heartbeat_timeout=idle_heartbeat_timeout,
idle_heartbeat_interval=idle_heartbeat_interval,
executor_threads=16)
session = cluster.connect(wait_for_all_pools=True)
cluster.register_listener(listener)
prime_request(PrimeOptions(then=NO_THEN))
prime_request(RejectConnections(RejectType.REJECT_STARTUP))
time.sleep((idle_heartbeat_timeout + idle_heartbeat_interval) * 2)
for host in cluster.metadata.all_hosts():
self.assertIn(host, listener.hosts_marked_down)
self.assertRaises(NoHostAvailable, session.execute, "SELECT * from system.local")
clear_queries()
prime_request(AcceptConnections())
time.sleep(idle_heartbeat_timeout + idle_heartbeat_interval + 2)
self.assertIsNotNone(session.execute("SELECT * from system.local"))
def test_idle_connection_is_not_closed(self):
"""
Test to ensure that the connections aren't closed if they are idle
@since 3.12
@jira_ticket PYTHON-573
@expected_result the connections aren't closed nor the hosts are
set to down if the connection is idle
@test_category connection
"""
start_and_prime_singledc()
idle_heartbeat_timeout = 1
idle_heartbeat_interval = 1
listener = TrackDownListener()
cluster = Cluster(compression=False,
idle_heartbeat_interval=idle_heartbeat_interval,
idle_heartbeat_timeout=idle_heartbeat_timeout)
session = cluster.connect(wait_for_all_pools=True)
cluster.register_listener(listener)
self.addCleanup(cluster.shutdown)
time.sleep(20)
self.assertEqual(listener.hosts_marked_down, [])
def test_max_in_flight(self):
""" Verify we don't exceed max_in_flight when borrowing connections or sending heartbeats """
Connection.max_in_flight = 50
start_and_prime_singledc()
profile = ExecutionProfile(request_timeout=1, load_balancing_policy=WhiteListRoundRobinPolicy(['127.0.0.1']))
cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
compression=False,
execution_profiles={EXEC_PROFILE_DEFAULT: profile},
idle_heartbeat_interval=.1,
idle_heartbeat_timeout=.1,
)
session = cluster.connect(wait_for_all_pools=True)
self.addCleanup(cluster.shutdown)
query = session.prepare("INSERT INTO table1 (id) VALUES (?)")
prime_request(PauseReads())
futures = []
# + 50 because simulacron doesn't immediately block all queries
for i in range(Connection.max_in_flight + 50):
futures.append(session.execute_async(query, ['a']))
prime_request(ResumeReads())
for future in futures:
# We're veryfing we don't get an assertion error from Connection.get_request_id,
# so skip any valid errors
try:
future.result()
except OperationTimedOut:
pass
except NoHostAvailable:
pass
def test_callbacks_and_pool_when_oto(self):
"""
Test to ensure the callbacks are correcltly called and the connection
is returned when there is an OTO
@since 3.12
@jira_ticket PYTHON-630
@expected_result the connection is correctly returned to the pool
after an OTO, also the only the errback is called and not the callback
when the message finally arrives.
@test_category metadata
"""
start_and_prime_singledc()
cluster = Cluster(protocol_version=PROTOCOL_VERSION, compression=False)
session = cluster.connect()
self.addCleanup(cluster.shutdown)
query_to_prime = "SELECT * from testkesypace.testtable"
server_delay = 2 # seconds
prime_query(query_to_prime, then={"delay_in_ms": server_delay * 1000})
future = session.execute_async(query_to_prime, timeout=1)
callback, errback = Mock(name='callback'), Mock(name='errback')
future.add_callbacks(callback, errback)
self.assertRaises(OperationTimedOut, future.result)
assert_quiescent_pool_state(self, cluster)
time.sleep(server_delay + 1)
# PYTHON-630 -- only the errback should be called
errback.assert_called_once()
callback.assert_not_called()
def test_close_when_query(self):
"""
Test to ensure the driver behaves correctly if the connection is closed
just when querying
@since 3.12
@expected_result NoHostAvailable is risen
@test_category connection
"""
start_and_prime_singledc()
cluster = Cluster(protocol_version=PROTOCOL_VERSION, compression=False)
session = cluster.connect()
self.addCleanup(cluster.shutdown)
query_to_prime = "SELECT * from testkesypace.testtable"
for close_type in ("disconnect", "shutdown_read", "shutdown_write"):
then = {
"result": "close_connection",
"delay_in_ms": 0,
"close_type": close_type,
"scope": "connection"
}
prime_query(query_to_prime, then=then, rows=None, column_types=None)
self.assertRaises(NoHostAvailable, session.execute, query_to_prime)
def test_close_when_query(self):
"""
Test to ensure the driver behaves correctly if the connection is closed
just when querying
@since 3.12
@expected_result NoHostAvailable is risen
@test_category connection
"""
start_and_prime_singledc()
cluster = Cluster(protocol_version=PROTOCOL_VERSION, compression=False)
session = cluster.connect()
self.addCleanup(cluster.shutdown)
query_to_prime = "SELECT * from testkesypace.testtable"
for close_type in ("disconnect", "shutdown_read", "shutdown_write"):
then = {
"result": "close_connection",
"delay_in_ms": 0,
"close_type": close_type,
"scope": "connection"
}
prime_query(query_to_prime,
then=then,
rows=None,
column_types=None)
self.assertRaises(NoHostAvailable, session.execute, query_to_prime)
def test_callbacks_and_pool_when_oto(self):
"""
Test to ensure the callbacks are correcltly called and the connection
is returned when there is an OTO
@since 3.12
@jira_ticket PYTHON-630
@expected_result the connection is correctly returned to the pool
after an OTO, also the only the errback is called and not the callback
when the message finally arrives.
@test_category metadata
"""
start_and_prime_singledc()
cluster = Cluster(protocol_version=PROTOCOL_VERSION, compression=False)
session = cluster.connect()
self.addCleanup(cluster.shutdown)
query_to_prime = "SELECT * from testkesypace.testtable"
server_delay = 2 # seconds
prime_query(query_to_prime, then={"delay_in_ms": server_delay * 1000})
future = session.execute_async(query_to_prime, timeout=1)
callback, errback = Mock(name='callback'), Mock(name='errback')
future.add_callbacks(callback, errback)
self.assertRaises(OperationTimedOut, future.result)
assert_quiescent_pool_state(self, cluster)
time.sleep(server_delay + 1)
# PYTHON-630 -- only the errback should be called
errback.assert_called_once()
callback.assert_not_called()
def test_idle_connection_is_not_closed(self):
"""
Test to ensure that the connections aren't closed if they are idle
@since 3.12
@jira_ticket PYTHON-573
@expected_result the connections aren't closed nor the hosts are
set to down if the connection is idle
@test_category connection
"""
start_and_prime_singledc()
idle_heartbeat_timeout = 1
idle_heartbeat_interval = 1
listener = TrackDownListener()
cluster = Cluster(compression=False,
idle_heartbeat_interval=idle_heartbeat_interval,
idle_heartbeat_timeout=idle_heartbeat_timeout)
session = cluster.connect(wait_for_all_pools=True)
cluster.register_listener(listener)
self.addCleanup(cluster.shutdown)
time.sleep(20)
self.assertEqual(listener.hosts_marked_down, [])
def test_node_busy(self):
""" Verify that once TCP buffer is full, queries continue to get re-routed to other nodes """
start_and_prime_singledc()
profile = ExecutionProfile(load_balancing_policy=RoundRobinPolicy())
cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
compression=False,
execution_profiles={EXEC_PROFILE_DEFAULT: profile},
)
session = cluster.connect(wait_for_all_pools=True)
self.addCleanup(cluster.shutdown)
query = session.prepare("INSERT INTO table1 (id) VALUES (?)")
prime_request(PauseReads(dc_id=0, node_id=0))
blocked_profile = ExecutionProfile(
load_balancing_policy=WhiteListRoundRobinPolicy(["127.0.0.1"]))
cluster.add_execution_profile('blocked_profile', blocked_profile)
# Fill our blocked node's tcp buffer until we get a busy exception
self._fill_buffers(session,
query,
expected_blocked=1,
execution_profile='blocked_profile')
# Now that our send buffer is completely full on one node,
# verify queries get re-routed to other nodes and queries complete successfully
for i in range(1000):
session.execute(query, [str(i)])
def test_host_is_not_set_to_down_after_query_oto(self):
"""
Test to ensure that the connections aren't closed if there's an
OperationTimedOut in a normal query. This should only happen from the
heart beat thread (in the case of a OperationTimedOut) with the default
configuration
@since 3.12
@expected_result the connections aren't closed nor the hosts are
set to down
@test_category connection
"""
start_and_prime_singledc()
query_to_prime = "SELECT * FROM madeup_keyspace.madeup_table"
prime_query(query_to_prime, then=NO_THEN)
listener = TrackDownListener()
cluster = Cluster(compression=False)
session = cluster.connect(wait_for_all_pools=True)
cluster.register_listener(listener)
futures = []
for _ in range(10):
future = session.execute_async(query_to_prime)
futures.append(future)
for f in futures:
f._event.wait()
self.assertIsInstance(f._final_exception, OperationTimedOut)
self.assertEqual(listener.hosts_marked_down, [])
assert_quiescent_pool_state(self, cluster)
def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < Version("2.1"):
return
start_and_prime_singledc()
cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION,
compression=False)
cls.session = cls.cluster.connect(wait_for_all_pools=True)
def test_can_shutdown_asyncoreconnection_subclass(self):
start_and_prime_singledc()
class ExtendedConnection(AsyncoreConnection):
pass
cluster = Cluster(contact_points=["127.0.0.2"],
connection_class=ExtendedConnection)
cluster.connect()
cluster.shutdown()
def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < "2.1":
return
start_and_prime_singledc()
cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION,
compression=False,
default_retry_policy=CustomRetryPolicy())
cls.session = cls.cluster.connect(wait_for_all_pools=True)
def test_can_shutdown_connection_subclass(self):
start_and_prime_singledc()
class ExtendedConnection(connection_class):
pass
cluster = Cluster(contact_points=["127.0.0.2"],
connection_class=ExtendedConnection)
cluster.connect()
cluster.shutdown()
def test_can_shutdown_connection_subclass(self):
start_and_prime_singledc()
class ExtendedConnection(connection_class):
pass
cluster = Cluster(protocol_version=PROTOCOL_VERSION,
contact_points=["127.0.0.2"],
connection_class=ExtendedConnection,
compression=False)
cluster.connect()
cluster.shutdown()
def test_paused_connections(self):
""" Verify all requests come back as expected if node resumes within query timeout """
start_and_prime_singledc()
profile = ExecutionProfile(request_timeout=500,
load_balancing_policy=RoundRobinPolicy())
cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
compression=False,
execution_profiles={EXEC_PROFILE_DEFAULT: profile},
)
session = cluster.connect(wait_for_all_pools=True)
self.addCleanup(cluster.shutdown)
query = session.prepare("INSERT INTO table1 (id) VALUES (?)")
prime_request(PauseReads())
futures = self._fill_buffers(session, query)
# Make sure we actually have some stuck in-flight requests
for in_flight in [
pool._connection.in_flight for pool in session.get_pools()
]:
self.assertGreater(in_flight, 100)
time.sleep(.5)
for in_flight in [
pool._connection.in_flight for pool in session.get_pools()
]:
self.assertGreater(in_flight, 100)
prime_request(ResumeReads())
for future in futures:
try:
future.result()
except NoHostAvailable as e:
# We shouldn't have any timeouts here, but all of the queries beyond what can fit
# in the tcp buffer will have returned with a ConnectionBusy exception
self.assertIn("ConnectionBusy", str(e))
# Verify that we can continue sending queries without any problems
for host in session.cluster.metadata.all_hosts():
session.execute(query, ["a"], host=host)
def test_queued_requests_timeout(self):
""" Verify that queued requests timeout as expected """
start_and_prime_singledc()
profile = ExecutionProfile(request_timeout=.1,
load_balancing_policy=RoundRobinPolicy())
cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
compression=False,
execution_profiles={EXEC_PROFILE_DEFAULT: profile},
)
session = cluster.connect(wait_for_all_pools=True)
self.addCleanup(cluster.shutdown)
query = session.prepare("INSERT INTO table1 (id) VALUES (?)")
prime_request(PauseReads())
futures = []
for i in range(1000):
future = session.execute_async(query, [str(i)])
future.add_callbacks(callback=self.callback_success,
errback=self.callback_error)
futures.append(future)
successes = 0
for future in futures:
try:
future.result()
successes += 1
except OperationTimedOut:
pass
# Simulacron will respond to a couple queries before cutting off reads, so we'll just verify
# that only "a few" successes happened here
self.assertLess(successes, 50)
self.assertLess(self.callback_successes, 50)
self.assertEqual(self.callback_errors,
len(futures) - self.callback_successes)
def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < Version("2.1"):
return
start_and_prime_singledc()
cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION, compression=False)
cls.session = cls.cluster.connect(wait_for_all_pools=True)
spec_ep_brr = ExecutionProfile(load_balancing_policy=BadRoundRobinPolicy(),
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(1, 6),
request_timeout=12)
spec_ep_rr = ExecutionProfile(speculative_execution_policy=ConstantSpeculativeExecutionPolicy(.5, 10),
request_timeout=12)
spec_ep_rr_lim = ExecutionProfile(load_balancing_policy=BadRoundRobinPolicy(),
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(0.5, 1),
request_timeout=12)
spec_ep_brr_lim = ExecutionProfile(load_balancing_policy=BadRoundRobinPolicy(),
speculative_execution_policy=ConstantSpeculativeExecutionPolicy(4, 10))
cls.cluster.add_execution_profile("spec_ep_brr", spec_ep_brr)
cls.cluster.add_execution_profile("spec_ep_rr", spec_ep_rr)
cls.cluster.add_execution_profile("spec_ep_rr_lim", spec_ep_rr_lim)
cls.cluster.add_execution_profile("spec_ep_brr_lim", spec_ep_brr_lim)
def test_cluster_busy(self):
""" Verify that once TCP buffer is full we get busy exceptions rather than timeouts """
start_and_prime_singledc()
profile = ExecutionProfile(load_balancing_policy=RoundRobinPolicy())
cluster = Cluster(
protocol_version=PROTOCOL_VERSION,
compression=False,
execution_profiles={EXEC_PROFILE_DEFAULT: profile},
)
session = cluster.connect(wait_for_all_pools=True)
self.addCleanup(cluster.shutdown)
query = session.prepare("INSERT INTO table1 (id) VALUES (?)")
prime_request(PauseReads())
# These requests will get stuck in the TCP buffer and we have no choice but to let them time out
self._fill_buffers(session, query, expected_blocked=3)
# Now that our send buffer is completely full, verify we immediately get busy exceptions rather than timing out
for i in range(1000):
with self.assertRaises(NoHostAvailable) as e:
session.execute(query, [str(i)])
self.assertIn("ConnectionBusy", str(e.exception))
def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < Version("2.1"):
return
start_and_prime_singledc()
def test_heartbeat_defunct_deadlock(self):
"""
Ensure that there is no deadlock when request is in-flight and heartbeat defuncts connection
@since 3.16
@jira_ticket PYTHON-1044
@expected_result an OperationTimeout is raised and no deadlock occurs
@test_category connection
"""
start_and_prime_singledc()
# This is all about timing. We will need the QUERY response future to time out and the heartbeat to defunct
# at the same moment. The latter will schedule a QUERY retry to another node in case the pool is not
# already shut down. If and only if the response future timeout falls in between the retry scheduling and
# its execution the deadlock occurs. The odds are low, so we need to help fate a bit:
# 1) Make one heartbeat messages be sent to every node
# 2) Our QUERY goes always to the same host
# 3) This host needs to defunct first
# 4) Open a small time window for the response future timeout, i.e. block executor threads for retry
# execution and last connection to defunct
query_to_prime = "SELECT * from testkesypace.testtable"
query_host = "127.0.0.2"
heartbeat_interval = 1
heartbeat_timeout = 1
lag = 0.05
never = 9999
class PatchedRoundRobinPolicy(RoundRobinPolicy):
# Send always to same host
def make_query_plan(self, working_keyspace=None, query=None):
if query and query.query_string == query_to_prime:
return filter(lambda h: h == query_host, self._live_hosts)
else:
return super(PatchedRoundRobinPolicy,
self).make_query_plan()
class PatchedCluster(Cluster):
# Make sure that QUERY connection will timeout first
def get_connection_holders(self):
holders = super(PatchedCluster, self).get_connection_holders()
return sorted(
holders,
reverse=True,
key=lambda v: int(v._connection.host == query_host))
# Block executor thread like closing a dead socket could do
def connection_factory(self, *args, **kwargs):
conn = super(PatchedCluster,
self).connection_factory(*args, **kwargs)
conn.defunct = late(seconds=2 * lag)(conn.defunct)
return conn
cluster = PatchedCluster(
protocol_version=PROTOCOL_VERSION,
compression=False,
idle_heartbeat_interval=heartbeat_interval,
idle_heartbeat_timeout=heartbeat_timeout,
load_balancing_policy=PatchedRoundRobinPolicy())
session = cluster.connect()
self.addCleanup(cluster.shutdown)
prime_query(query_to_prime, then={"delay_in_ms": never})
# Make heartbeat due
time.sleep(heartbeat_interval)
future = session.execute_async(query_to_prime,
timeout=heartbeat_interval +
heartbeat_timeout + 3 * lag)
# Delay thread execution like kernel could do
future._retry_task = late(seconds=4 * lag)(future._retry_task)
prime_request(
PrimeOptions(then={
"result": "no_result",
"delay_in_ms": never
}))
prime_request(RejectConnections("unbind"))
self.assertRaisesRegexp(OperationTimedOut,
"Connection defunct by heartbeat",
future.result)
def setUpClass(cls):
if SIMULACRON_JAR is None or CASSANDRA_VERSION < "2.1":
return
start_and_prime_singledc()
