def test_replace_with_insufficient_replicas(self):
"""
Test that replace fails when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
self.fixture_dtest_setup.ignore_log_patterns = list(
self.fixture_dtest_setup.ignore_log_patterns) + [
r'Unable to find sufficient sources for streaming range'
]
self._setup(n=3)
self._insert_data(rf=2)
self._stop_node_to_replace()
# stop other replica
logger.debug("Stopping other replica")
self.query_node.stop(wait_other_notice=True)
self._do_replace(wait_for_binary_proto=False, wait_other_notice=False)
# replace should fail due to insufficient replicas
self.replacement_node.watch_log_for(
"Unable to find sufficient sources for streaming range")
assert_not_running(self.replacement_node)
def replace_active_node_test(self):
self.ignore_log_patterns = list(self.ignore_log_patterns) + [r'Exception encountered during startup']
self._setup(n=3)
self._do_replace(wait_for_binary_proto=False)
debug("Waiting for replace to fail")
self.replacement_node.watch_log_for("java.lang.UnsupportedOperationException: Cannot replace a live node...")
assert_not_running(self.replacement_node)
def replace_active_node_test(self):
self.ignore_log_patterns = list(self.ignore_log_patterns) + [r'Exception encountered during startup']
self._setup(n=3)
self._do_replace(wait_for_binary_proto=False)
debug("Waiting for replace to fail")
self.replacement_node.watch_log_for("java.lang.UnsupportedOperationException: Cannot replace a live node...")
assert_not_running(self.replacement_node)
def _bootstrap_test_with_replica_down(self, consistent_range_movement, rf=2):
"""
Test to check consistent bootstrap will not succeed when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
cluster = self.cluster
cluster.set_environment_variable('CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2)
node1, node2 = cluster.nodelist()
node3_token = None
# Make token assignment deterministic
if not self.dtest_config.use_vnodes:
cluster.set_configuration_options(values={'num_tokens': 1})
tokens = cluster.balanced_tokens(3)
logger.debug("non-vnode tokens: %r" % (tokens,))
node1.set_configuration_options(values={'initial_token': tokens[0]})
node2.set_configuration_options(values={'initial_token': tokens[2]})
node3_token = tokens[1] # Add node 3 between node1 and node2
cluster.start()
node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema', 'replication(factor={})'.format(rf)])
# change system_auth keyspace to 2 (default is 1) to avoid
# "Unable to find sufficient sources for streaming" warning
if cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(node1)
session.execute("""
ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};
""")
# Stop node2, so node3 will not be able to perform consistent range movement
node2.stop(wait_other_notice=True)
successful_bootstrap_expected = not consistent_range_movement
node3 = new_node(cluster, token=node3_token)
node3.start(wait_for_binary_proto=successful_bootstrap_expected, wait_other_notice=successful_bootstrap_expected,
jvm_args=["-Dcassandra.consistent.rangemovement={}".format(consistent_range_movement)])
if successful_bootstrap_expected:
# with rf=1 and cassandra.consistent.rangemovement=false, missing sources are ignored
if not consistent_range_movement and rf == 1:
node3.watch_log_for("Unable to find sufficient sources for streaming range")
assert node3.is_running()
assert_bootstrap_state(self, node3, 'COMPLETED')
else:
if consistent_range_movement:
if cluster.version() < '4.0':
node3.watch_log_for("A node required to move the data consistently is down")
else:
node3.watch_log_for("Necessary replicas for strict consistency were removed by source filters")
else:
node3.watch_log_for("Unable to find sufficient sources for streaming range")
assert_not_running(node3)
def _bootstrap_test_with_replica_down(self, consistent_range_movement, rf=2):
"""
Test to check consistent bootstrap will not succeed when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
cluster = self.cluster
cluster.populate(2)
node1, node2 = cluster.nodelist()
node3_token = None
# Make token assignment deterministic
if not self.dtest_config.use_vnodes:
cluster.set_configuration_options(values={'num_tokens': 1})
tokens = cluster.balanced_tokens(3)
logger.debug("non-vnode tokens: %r" % (tokens,))
node1.set_configuration_options(values={'initial_token': tokens[0]})
node2.set_configuration_options(values={'initial_token': tokens[2]})
node3_token = tokens[1] # Add node 3 between node1 and node2
cluster.start()
node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema', 'replication(factor={})'.format(rf)])
# change system_auth keyspace to 2 (default is 1) to avoid
# "Unable to find sufficient sources for streaming" warning
if cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(node1)
session.execute("""
ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};
""")
# Stop node2, so node3 will not be able to perform consistent range movement
node2.stop(wait_other_notice=True)
successful_bootstrap_expected = not consistent_range_movement
node3 = new_node(cluster, token=node3_token)
node3.start(wait_for_binary_proto=successful_bootstrap_expected, wait_other_notice=successful_bootstrap_expected,
jvm_args=["-Dcassandra.consistent.rangemovement={}".format(consistent_range_movement)])
if successful_bootstrap_expected:
# with rf=1 and cassandra.consistent.rangemovement=false, missing sources are ignored
if not consistent_range_movement and rf == 1:
node3.watch_log_for("Unable to find sufficient sources for streaming range")
assert node3.is_running()
assert_bootstrap_state(self, node3, 'COMPLETED')
else:
if consistent_range_movement:
if cluster.version() < '4.0':
node3.watch_log_for("A node required to move the data consistently is down")
else:
node3.watch_log_for("Necessary replicas for strict consistency were removed by source filters")
else:
node3.watch_log_for("Unable to find sufficient sources for streaming range")
assert_not_running(node3)
def replace_nonexistent_node_test(self):
self.ignore_log_patterns = list(self.ignore_log_patterns) + [
# This is caused by starting a node improperly (replacing active/nonexistent)
r'Exception encountered during startup',
# This is caused by trying to replace a nonexistent node
r'Exception in thread Thread']
self._setup(n=3)
self._do_replace(replace_address='127.0.0.5', wait_for_binary_proto=False)
debug("Waiting for replace to fail")
self.replacement_node.watch_log_for("java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip")
assert_not_running(self.replacement_node)
def replace_nonexistent_node_test(self):
self.ignore_log_patterns = list(self.ignore_log_patterns) + [
# This is caused by starting a node improperly (replacing active/nonexistent)
r'Exception encountered during startup',
# This is caused by trying to replace a nonexistent node
r'Exception in thread Thread']
self._setup(n=3)
self._do_replace(replace_address='127.0.0.5', wait_for_binary_proto=False)
debug("Waiting for replace to fail")
self.replacement_node.watch_log_for("java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip")
assert_not_running(self.replacement_node)
def test_replace_nonexistent_node(self):
self.fixture_dtest_setup.ignore_log_patterns = list(self.fixture_dtest_setup.ignore_log_patterns) + [
# This is caused by starting a node improperly (replacing active/nonexistent)
r'Exception encountered during startup',
# This is caused by trying to replace a nonexistent node
r'Exception in thread Thread']
self._setup(n=3)
self._do_replace(replace_address='127.0.0.5', wait_for_binary_proto=False)
logger.debug("Waiting for replace to fail")
node_log_str = "/127.0.0.5" if self.cluster.version() < '4.0' else "/127.0.0.5:7000"
self.replacement_node.watch_log_for("java.lang.RuntimeException: Cannot replace_address "
+ node_log_str + " because it doesn't exist in gossip")
assert_not_running(self.replacement_node)
def replace_with_insufficient_replicas_test(self):
"""
Test that replace fails when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
self.ignore_log_patterns = list(self.ignore_log_patterns) + [r'Unable to find sufficient sources for streaming range']
self._setup(n=3)
self._insert_data(rf=2)
self._stop_node_to_replace()
# stop other replica
debug("Stopping other replica")
self.query_node.stop(wait_other_notice=True)
self._do_replace(wait_for_binary_proto=False, wait_other_notice=False)
# replace should fail due to insufficient replicas
self.replacement_node.watch_log_for("Unable to find sufficient sources for streaming range")
assert_not_running(self.replacement_node)
