def test_write_timeout(self):
policy = FallthroughRetryPolicy()
retry, consistency = policy.on_write_timeout(
query=None,
consistency="ONE",
write_type=WriteType.SIMPLE,
required_responses=1,
received_responses=2,
retry_num=1)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_write_timeout(
query=None,
consistency="ONE",
write_type=WriteType.SIMPLE,
required_responses=1,
received_responses=2,
retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_write_timeout(
query=None,
consistency="ONE",
write_type=WriteType.BATCH_LOG,
required_responses=10000,
received_responses=1,
retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
def test_unavailable(self):
policy = FallthroughRetryPolicy()
retry, consistency = policy.on_unavailable(query=None,
consistency="ONE",
required_replicas=1,
alive_replicas=2,
retry_num=1)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_unavailable(query=None,
consistency="ONE",
required_replicas=1,
alive_replicas=2,
retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_unavailable(query=None,
consistency="ONE",
required_replicas=10000,
alive_replicas=1,
retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
def test_read_timeout(self):
policy = FallthroughRetryPolicy()
retry, consistency = policy.on_read_timeout(
query=None, consistency=ONE, required_responses=1, received_responses=2,
data_retrieved=True, retry_num=1)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_read_timeout(
query=None, consistency=ONE, required_responses=2, received_responses=1,
data_retrieved=True, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_read_timeout(
query=None, consistency=ONE, required_responses=2, received_responses=2,
data_retrieved=True, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_read_timeout(
query=None, consistency=ONE, required_responses=2, received_responses=2,
data_retrieved=False, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
def test_write_timeout(self):
policy = FallthroughRetryPolicy()
retry, consistency = policy.on_write_timeout(
query=None,
consistency=ONE,
write_type=WriteType.SIMPLE,
required_responses=1,
received_responses=2,
retry_num=1,
)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_write_timeout(
query=None,
consistency=ONE,
write_type=WriteType.SIMPLE,
required_responses=1,
received_responses=2,
retry_num=0,
)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_write_timeout(
query=None,
consistency=ONE,
write_type=WriteType.BATCH_LOG,
required_responses=10000,
received_responses=1,
retry_num=0,
)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
def test_duplicate_metrics_per_cluster(self):
"""
Test to validate that cluster metrics names can't overlap.
@since 3.6.0
@jira_ticket PYTHON-561
@expected_result metric names should not be allowed to be same.
@test_category metrics
"""
cluster2 = Cluster(metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
default_retry_policy=FallthroughRetryPolicy())
cluster3 = Cluster(metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
default_retry_policy=FallthroughRetryPolicy())
# Ensure duplicate metric names are not allowed
cluster2.metrics.set_stats_name("appcluster")
cluster2.metrics.set_stats_name("appcluster")
with self.assertRaises(ValueError):
cluster3.metrics.set_stats_name("appcluster")
cluster3.metrics.set_stats_name("devops")
session2 = cluster2.connect(self.ks_name, wait_for_all_pools=True)
session3 = cluster3.connect(self.ks_name, wait_for_all_pools=True)
# Basic validation that naming metrics doesn't impact their segration or accuracy
for i in range(10):
query = SimpleStatement("SELECT * FROM {0}.{0}".format(
self.ks_name),
consistency_level=ConsistencyLevel.ALL)
session2.execute(query)
for i in range(5):
query = SimpleStatement("SELECT * FROM {0}.{0}".format(
self.ks_name),
consistency_level=ConsistencyLevel.ALL)
session3.execute(query)
self.assertEqual(
cluster2.metrics.get_stats()['request_timer']['count'], 10)
self.assertEqual(
cluster3.metrics.get_stats()['request_timer']['count'], 5)
# Check scales to ensure they are appropriately named
self.assertTrue("appcluster" in scales._Stats.stats.keys())
self.assertTrue("devops" in scales._Stats.stats.keys())
cluster2.shutdown()
cluster3.shutdown()
def __init__(self, log_level=logging.INFO):
"""
Constructor.
"""
class_name = self.__class__.__name__
self.logger = logging.getLogger(class_name)
self.logger.setLevel(log_level)
self.logger.info('Starting {}'.format(class_name))
self.hosts = appscale_info.get_db_ips()
self.retry_policy = IdempotentRetryPolicy()
self.no_retries = FallthroughRetryPolicy()
remaining_retries = INITIAL_CONNECT_RETRIES
while True:
try:
self.cluster = Cluster(self.hosts,
default_retry_policy=self.retry_policy)
self.session = self.cluster.connect(KEYSPACE)
break
except cassandra.cluster.NoHostAvailable as connection_error:
remaining_retries -= 1
if remaining_retries < 0:
raise connection_error
time.sleep(3)
self.session.default_consistency_level = ConsistencyLevel.QUORUM
def index_query_test(self):
"""
Check that a secondary index query times out
"""
cluster = self.cluster
cluster.set_configuration_options(values={'read_request_timeout_in_ms': 1000})
cluster.populate(1).start(wait_for_binary_proto=True, jvm_args=["-Dcassandra.monitoring_check_interval_ms=50",
"-Dcassandra.test.read_iteration_delay_ms=1500"]) # see above for explanation
node = cluster.nodelist()[0]
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
session.execute("""
CREATE TABLE test3 (
id int PRIMARY KEY,
col int,
val text
);
""")
session.execute("CREATE INDEX ON test3 (col)")
for i in xrange(500):
session.execute("INSERT INTO test3 (id, col, val) VALUES ({}, {}, 'foo')".format(i, i // 10))
mark = node.mark_log()
statement = session.prepare("SELECT * from test3 WHERE col < ? ALLOW FILTERING")
statement.consistency_level = ConsistencyLevel.ONE
statement.retry_policy = FallthroughRetryPolicy()
assert_unavailable(lambda c: debug(c.execute(statement, [50])), session)
node.watch_log_for("Some operations timed out", from_mark=mark, timeout=60)
def test_metrics_per_cluster(self):
"""
Test to validate that metrics can be scopped to invdividual clusters
@since 3.6.0
@jira_ticket PYTHON-561
@expected_result metrics should be scopped to a cluster level
@test_category metrics
"""
cluster2 = Cluster(metrics_enabled=True,
protocol_version=PROTOCOL_VERSION,
default_retry_policy=FallthroughRetryPolicy())
cluster2.connect(self.ks_name, wait_for_all_pools=True)
self.assertEqual(len(cluster2.metadata.all_hosts()), 3)
query = SimpleStatement("SELECT * FROM {0}.{0}".format(self.ks_name),
consistency_level=ConsistencyLevel.ALL)
self.session.execute(query)
# Pause node so it shows as unreachable to coordinator
get_node(1).pause()
try:
# Test write
query = SimpleStatement(
"INSERT INTO {0}.{0} (k, v) VALUES (2, 2)".format(
self.ks_name),
consistency_level=ConsistencyLevel.ALL)
with self.assertRaises(WriteTimeout):
self.session.execute(query, timeout=None)
finally:
get_node(1).resume()
# Change the scales stats_name of the cluster2
cluster2.metrics.set_stats_name('cluster2-metrics')
stats_cluster1 = self.cluster.metrics.get_stats()
stats_cluster2 = cluster2.metrics.get_stats()
# Test direct access to stats
self.assertEqual(1, self.cluster.metrics.stats.write_timeouts)
self.assertEqual(0, cluster2.metrics.stats.write_timeouts)
# Test direct access to a child stats
self.assertNotEqual(0.0, self.cluster.metrics.request_timer['mean'])
self.assertEqual(0.0, cluster2.metrics.request_timer['mean'])
# Test access via metrics.get_stats()
self.assertNotEqual(0.0, stats_cluster1['request_timer']['mean'])
self.assertEqual(0.0, stats_cluster2['request_timer']['mean'])
# Test access by stats_name
self.assertEqual(
0.0,
scales.getStats()['cluster2-metrics']['request_timer']['mean'])
cluster2.shutdown()
def _insert_tombstones(self, session, number_of_tombstones):
for num_id in range(number_of_tombstones):
session.execute(
SimpleStatement(
"DELETE value FROM tombstonefailure WHERE id = {}".format(
num_id),
consistency_level=self.consistency_level,
retry_policy=FallthroughRetryPolicy()))
def setUp(self):
contact_point = ['127.0.0.2']
self.cluster = Cluster(contact_points=contact_point, metrics_enabled=True, protocol_version=PROTOCOL_VERSION,
load_balancing_policy=HostFilterPolicy(
RoundRobinPolicy(), lambda host: host.address in contact_point
),
default_retry_policy=FallthroughRetryPolicy())
self.session = self.cluster.connect("test3rf", wait_for_all_pools=True)
def create(self, location):
""" Saves a location in the database
Args:
location: model.Location to insert into database
"""
query = """
INSERT INTO location (
username,
latitude,
longitude,
timestamp_created,
location_name
) VALUES (
%(username)s,
%(latitude)s,
%(longitude)s,
%(timestamp_created)s,
%(location_name)s
)
"""
location_dict = location.to_dict()
client = current_app.extensions['registry']['CASSANDRA_CLIENT']
client.execute(query,
params=location_dict,
routing_key=location.username,
retry_policy=FallthroughRetryPolicy())
query = """
INSERT INTO location_by_timestamp (
username,
latitude,
longitude,
timestamp_created,
location_name
) VALUES (
%(username)s,
%(latitude)s,
%(longitude)s,
%(timestamp_created)s,
%(location_name)s
)
"""
client.execute(query,
params=location_dict,
routing_key=location.username,
retry_policy=FallthroughRetryPolicy())
def materialized_view_test(self):
"""
Check that a materialized view query times out:
- populate a 2-node cluster
- set a 1-second read timeout
- start one node without having it join the ring
- start the other node with read_iteration_delay set to 1.5 seconds
- (this will cause read queries to take longer than the read timeout)
- CREATE a table
- CREATE a materialized view over that table
- INSERT 50 values into that table
- assert querying that table results in an unavailable exception
"""
cluster = self.cluster
cluster.set_configuration_options(
values={'read_request_timeout_in_ms': 1000})
cluster.populate(2)
node1, node2 = cluster.nodelist()
node1.start(wait_for_binary_proto=True,
join_ring=False) # ensure other node executes queries
node2.start(wait_for_binary_proto=True,
jvm_args=[
"-Dcassandra.monitoring_check_interval_ms=50",
"-Dcassandra.test.read_iteration_delay_ms=1500"
]) # see above for explanation
session = self.patient_exclusive_cql_connection(node1)
self.create_ks(session, 'ks', 1)
session.execute("""
CREATE TABLE test4 (
id int PRIMARY KEY,
col int,
val text
);
""")
session.execute(
("CREATE MATERIALIZED VIEW mv AS SELECT * FROM test4 "
"WHERE col IS NOT NULL AND id IS NOT NULL PRIMARY KEY (col, id)"))
for i in range(50):
session.execute(
"INSERT INTO test4 (id, col, val) VALUES ({}, {}, 'foo')".
format(i, i // 10))
mark = node2.mark_log()
statement = SimpleStatement("SELECT * FROM mv WHERE col = 50",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
node2.watch_log_for("operations timed out", from_mark=mark, timeout=60)
def test_unavailable(self):
policy = FallthroughRetryPolicy()
retry, consistency = policy.on_unavailable(
query=None, consistency=ONE,
required_replicas=1, alive_replicas=2, retry_num=1)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_unavailable(
query=None, consistency=ONE,
required_replicas=1, alive_replicas=2, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_unavailable(
query=None, consistency=ONE,
required_replicas=10000, alive_replicas=1, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
def test_read_timeout(self):
policy = FallthroughRetryPolicy()
retry, consistency = policy.on_read_timeout(
query=None, consistency="ONE", required_responses=1, received_responses=2,
data_retrieved=True, retry_num=1)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_read_timeout(
query=None, consistency="ONE", required_responses=2, received_responses=1,
data_retrieved=True, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_read_timeout(
query=None, consistency="ONE", required_responses=2, received_responses=2,
data_retrieved=True, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
retry, consistency = policy.on_read_timeout(
query=None, consistency="ONE", required_responses=2, received_responses=2,
data_retrieved=False, retry_num=0)
self.assertEqual(retry, RetryPolicy.RETHROW)
self.assertEqual(consistency, None)
def _perform_cql_statement(self, session, text_statement):
statement = SimpleStatement(text_statement,
consistency_level=self.consistency_level,
retry_policy=FallthroughRetryPolicy())
if self.expected_expt is None:
session.execute(statement)
else:
with pytest.raises(self.expected_expt) as cm:
# On 2.1, we won't return the ReadTimeout from coordinator until actual timeout,
# so we need to up the default timeout of the driver session
session.execute(statement, timeout=15)
return cm._excinfo[1]
def setUp(self):
contact_point = ['127.0.0.2']
self.cluster = TestCluster(contact_points=contact_point, metrics_enabled=True,
execution_profiles=
{EXEC_PROFILE_DEFAULT:
ExecutionProfile(
load_balancing_policy=HostFilterPolicy(
RoundRobinPolicy(), lambda host: host.address in contact_point),
retry_policy=FallthroughRetryPolicy()
)
}
)
self.session = self.cluster.connect("test3rf", wait_for_all_pools=True)
def local_query_test(self):
"""
Check that a query running on the local coordinator node times out:
- set a 1-second read timeout
- start the cluster with read_iteration_delay set to 1.5 seconds
- (this will cause read queries to take longer than the read timeout)
- CREATE and INSERT into a table
- SELECT * from the table using a retry policy that never retries, and assert it times out
@jira_ticket CASSANDRA-7392
"""
cluster = self.cluster
cluster.set_configuration_options(
values={'read_request_timeout_in_ms': 1000})
# cassandra.test.read_iteration_delay_ms causes the state tracking read iterators
# introduced by CASSANDRA-7392 to pause by the specified amount of milliseconds during each
# iteration of non system queries, so that these queries take much longer to complete,
# see ReadCommand.withStateTracking()
cluster.populate(1).start(
wait_for_binary_proto=True,
jvm_args=[
"-Dcassandra.monitoring_check_interval_ms=50",
"-Dcassandra.test.read_iteration_delay_ms=1500"
])
node = cluster.nodelist()[0]
session = self.patient_cql_connection(node)
self.create_ks(session, 'ks', 1)
session.execute("""
CREATE TABLE test1 (
id int PRIMARY KEY,
val text
);
""")
for i in range(500):
session.execute(
"INSERT INTO test1 (id, val) VALUES ({}, 'foo')".format(i))
mark = node.mark_log()
statement = SimpleStatement("SELECT * from test1",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
node.watch_log_for("operations timed out", from_mark=mark, timeout=60)
def setup_once(self):
cluster = self.cluster
cluster.set_configuration_options({
'read_request_timeout_in_ms': 3000,
'write_request_timeout_in_ms': 3000,
'phi_convict_threshold': 12,
'tombstone_failure_threshold': TOMBSTONE_FAILURE_THRESHOLD,
'enable_materialized_views': 'true'
})
cluster.populate(2, debug=True)
cluster.start(jvm_args=JVM_ARGS)
node1 = cluster.nodelist()[0]
global jmx
jmx = JolokiaAgent(node1)
jmx.start()
s = self.session = self.patient_exclusive_cql_connection(
node1, retry_policy=FallthroughRetryPolicy(), request_timeout=30)
for k in [KEYSPACE, FAIL_WRITE_KEYSPACE]:
create_ks(s, k, 2)
s.execute(
f"CREATE TABLE {k}.{TABLE} (k int, c int, v int, PRIMARY KEY (k,c))"
)
create_ks(s, VIEW_KEYSPACE, 1)
s.execute(
f"CREATE TABLE {VIEW_KEYSPACE}.{TABLE} (k int, c int, v int, PRIMARY KEY (k,c))"
)
s.execute(
f"CREATE MATERIALIZED VIEW {VIEW_KEYSPACE}.{VIEW} AS SELECT * FROM {TABLE} WHERE c IS NOT NULL AND k IS NOT NULL PRIMARY KEY (c,k);"
)
# Here we're doing a series of deletions in order to create enough tombstones to exceed the configured fail threshold.
# This partition will be used to test read failures.
for c in range(TOMBSTONE_FAILURE_THRESHOLD + 1):
self.session.execute(
f"DELETE FROM {KEYSPACE}.{TABLE} WHERE k={TOMBSTONE_FAIL_KEY} AND c={c}"
)
node1.watch_log_for(
"Created default superuser role 'cassandra'"
) # don't race with async default role creation, which creates a write
node1.watch_log_for(
'Completed submission of build tasks for any materialized views defined at startup',
filename='debug.log') # view builds cause background reads
def remote_query_test(self):
"""
Check that a query running on a node other than the coordinator times out:
- populate the cluster with 2 nodes
- set a 1-second read timeout
- start one node without having it join the ring
- start the other node with read_iteration_delay set to 1.5 seconds
- (this will cause read queries to take longer than the read timeout)
- CREATE a table
- INSERT 5000 rows on a session on the node that is not a member of the ring
- run SELECT statements and assert they fail
# TODO refactor SELECT statements:
# - run the statements in a loop to reduce duplication
# - watch the log after each query
# - assert we raise the right error
"""
cluster = self.cluster
cluster.set_configuration_options(
values={'read_request_timeout_in_ms': 1000})
cluster.populate(2)
node1, node2 = cluster.nodelist()
node1.start(wait_for_binary_proto=True,
join_ring=False) # ensure other node executes queries
node2.start(wait_for_binary_proto=True,
jvm_args=[
"-Dcassandra.monitoring_check_interval_ms=50",
"-Dcassandra.test.read_iteration_delay_ms=1500"
]) # see above for explanation
session = self.patient_exclusive_cql_connection(node1)
self.create_ks(session, 'ks', 1)
session.execute("""
CREATE TABLE test2 (
id int,
col int,
val text,
PRIMARY KEY(id, col)
);
""")
for i, j in itertools.product(range(500), range(10)):
session.execute(
"INSERT INTO test2 (id, col, val) VALUES ({}, {}, 'foo')".
format(i, j))
mark = node2.mark_log()
statement = SimpleStatement("SELECT * from test2",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement("SELECT * from test2 where id = 1",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement(
"SELECT * from test2 where id IN (1, 10, 20) AND col < 10",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement(
"SELECT * from test2 where col > 5 ALLOW FILTERING",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
node2.watch_log_for("operations timed out", from_mark=mark, timeout=60)
from cassandra.cluster import SimpleStatement
from cassandra.policies import FallthroughRetryPolicy, RetryPolicy
from .cassandra_interface import IndexStates
from .cassandra_interface import INITIAL_CONNECT_RETRIES
from .cassandra_interface import KEYSPACE
from .cassandra_interface import ScatterPropStates
from .cassandra_interface import ThriftColumn
from .constants import CURRENT_VERSION, LB_POLICY
from .. import dbconstants
sys.path.append(APPSCALE_PYTHON_APPSERVER)
from google.appengine.datastore import entity_pb
from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
# A policy that does not retry statements.
NO_RETRIES = FallthroughRetryPolicy()
logger = logging.getLogger(__name__)
# The number of times to retry idempotent statements.
BASIC_RETRY_COUNT = 5
class IdempotentRetryPolicy(RetryPolicy):
""" A policy used for retrying idempotent statements. """
def on_read_timeout(self, query, consistency, required_responses,
received_responses, data_retrieved, retry_num):
""" This is called when a ReadTimeout occurs.
Args:
query: A statement that timed out.
def remote_query_test(self):
"""
Check that a query running on a node other than the coordinator times out
"""
cluster = self.cluster
cluster.set_configuration_options(
values={'read_request_timeout_in_ms': 1000})
cluster.populate(2)
node1, node2 = cluster.nodelist()
node1.start(wait_for_binary_proto=True,
join_ring=False) # ensure other node executes queries
node2.start(wait_for_binary_proto=True,
jvm_args=[
"-Dcassandra.monitoring_check_interval_ms=50",
"-Dcassandra.test.read_iteration_delay_ms=1500"
]) # see above for explanation
session = self.patient_exclusive_cql_connection(node1)
self.create_ks(session, 'ks', 1)
session.execute("""
CREATE TABLE test2 (
id int,
col int,
val text,
PRIMARY KEY(id, col)
);
""")
for i in xrange(500):
for j in xrange(10):
session.execute(
"INSERT INTO test2 (id, col, val) VALUES ({}, {}, 'foo')".
format(i, j))
mark = node2.mark_log()
statement = SimpleStatement("SELECT * from test2",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement("SELECT * from test2 where id = 1",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement(
"SELECT * from test2 where id IN (1, 10, 20) AND col < 10",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
statement = SimpleStatement(
"SELECT * from test2 where col > 5 ALLOW FILTERING",
consistency_level=ConsistencyLevel.ONE,
retry_policy=FallthroughRetryPolicy())
assert_unavailable(lambda c: debug(c.execute(statement)), session)
node2.watch_log_for("Some operations timed out",
from_mark=mark,
timeout=60)
