def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
mark = node4.mark_log()
node4.start(replace_address='127.0.0.3', wait_other_notice=False)
node4.watch_log_for(
"java.lang.UnsupportedOperationException: Cannot replace a live node...",
from_mark=mark)
assert_not_running(node4)
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
# try to replace an unassigned ip address
mark = node4.mark_log()
node4.start(replace_address='127.0.0.5', wait_other_notice=False)
node4.watch_log_for(
"java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip",
from_mark=mark)
assert_not_running(node4)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def resumable_replace_test(self):
"""Test resumable bootstrap while replacing node"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster,
True, ('127.0.0.4', 9160), ('127.0.0.4', 7000),
'7400',
'0',
None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"])
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = list(
session.execute(
"SELECT bootstrapped FROM system.local WHERE key='local'"))
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = list(session.execute("SELECT bootstrapped FROM system.local WHERE key='local'"))
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = list(session.execute("SELECT bootstrapped FROM system.local WHERE key='local'"))
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
def replace_with_insufficient_replicas_test(self):
"""
Test that replace fails when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress([
'write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=2)'
])
# stop node to replace
debug("Stopping node to replace.")
node3.stop(wait_other_notice=True)
# stop other replica
debug("Stopping node2 (other replica)")
node2.stop(wait_other_notice=True)
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3',
wait_for_binary_proto=False,
wait_other_notice=False)
# replace should fail due to insufficient replicas
node4.watch_log_for(
"Unable to find sufficient sources for streaming range")
assert_not_running(node4)
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
# try to replace an unassigned ip address
mark = node4.mark_log()
node4.start(replace_address='127.0.0.5', wait_other_notice=False)
node4.watch_log_for("java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip", from_mark=mark)
self.check_not_running(node4)
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
mark = node4.mark_log()
node4.start(replace_address='127.0.0.3', wait_other_notice=False)
node4.watch_log_for("java.lang.UnsupportedOperationException: Cannot replace a live node...", from_mark=mark)
self.check_not_running(node4)
def resumable_replace_test(self):
"""Test resumable bootstrap while replacing node"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version() >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = session.execute(query)
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, ('127.0.0.4', 9042))
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"])
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = session.execute("SELECT bootstrapped FROM system.local WHERE key='local'")
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
#query should work again
debug("Verifying querying works again.")
finalData = session.execute(query)
self.assertListEqual(initialData, finalData)
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
mark = node4.mark_log()
node4.start(replace_address='127.0.0.3', wait_other_notice=False)
node4.watch_log_for("java.lang.UnsupportedOperationException: Cannot replace a live node...", from_mark=mark)
assert_not_running(node4)
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
# try to replace an unassigned ip address
mark = node4.mark_log()
node4.start(replace_address='127.0.0.5', wait_other_notice=False)
node4.watch_log_for("java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip", from_mark=mark)
assert_not_running(node4)
def replace_with_insufficient_replicas_test(self):
"""
Test that replace fails when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress(['write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=2)'])
# stop node to replace
debug("Stopping node to replace.")
node3.stop(wait_other_notice=True)
# stop other replica
debug("Stopping node2 (other replica)")
node2.stop(wait_other_notice=True)
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=False, wait_other_notice=False)
# replace should fail due to insufficient replicas
node4.watch_log_for("Unable to find sufficient sources for streaming range")
assert_not_running(node4)
def test_upgrade_legacy_table(self):
"""
Upgrade with bringing up the legacy tables after the newer nodes (without legacy tables)
were started.
@jira_ticket CASSANDRA-12813
"""
cluster = self.cluster
# Forcing cluster version on purpose
cluster.set_install_dir(version="2.1.16")
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# Wait for default user to get created on one of the nodes
time.sleep(15)
# Upgrade to current version
for node in [node1, node2, node3]:
node.drain()
node.watch_log_for("DRAINED")
node.stop(gently=True)
self.set_node_to_current_version(node)
cluster.start()
# Make sure the system_auth table will get replicated to the node that we're going to replace
session = self.patient_cql_connection(node1, user='******', password='******')
session.execute("ALTER KEYSPACE system_auth WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 };")
cluster.repair()
cluster.stop()
# Replace the node
cluster.seeds.remove(node1)
cluster.remove(node1)
replacement_address = node1.address()
replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=(replacement_address, 9160), storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042))
self.set_node_to_current_version(replacement_node)
cluster.add(replacement_node, True)
replacement_node.start(wait_for_binary_proto=True)
node2.start(wait_for_binary_proto=True)
node3.start(wait_for_binary_proto=True)
replacement_node.watch_log_for('Initializing system_auth.credentials')
replacement_node.watch_log_for('Initializing system_auth.permissions')
replacement_node.watch_log_for('Initializing system_auth.users')
cluster.repair()
replacement_node.watch_log_for('Repair command')
# Should succeed. Will throw an NPE on pre-12813 code.
self.patient_cql_connection(replacement_node, user='******', password='******')
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress([
'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(
values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"],
wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh",
from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
class BaseReplaceAddressTest(Tester):
@pytest.fixture(autouse=True)
def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
fixture_dtest_setup.ignore_log_patterns = (
# This one occurs when trying to send the migration to a
# node that hasn't started yet, and when it does, it gets
# replayed and everything is fine.
r'Can\'t send migration request: node.*is down',
r'Migration task failed to complete', # 10978
# ignore streaming error during bootstrap
r'Streaming error occurred',
r'failed stream session',
r'Failed to properly handshake with peer'
)
def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False):
logger.debug("Starting cluster with {} nodes.".format(n))
self.cluster.populate(n)
if opts is not None:
logger.debug("Setting cluster options: {}".format(opts))
self.cluster.set_configuration_options(opts)
self.cluster.set_batch_commitlog(enabled=True)
self.query_node = self.cluster.nodelist()[0]
self.replaced_node = self.cluster.nodelist()[-1]
self.cluster.seeds.remove(self.replaced_node)
NUM_TOKENS = os.environ.get('NUM_TOKENS', '256')
if not self.dtest_config.use_vnodes:
self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': 1})
else:
self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': NUM_TOKENS})
if enable_byteman:
# set up byteman
self.query_node.byteman_port = '8100'
self.query_node.import_config_files()
if mixed_versions:
logger.debug("Starting nodes on version 2.2.4")
self.cluster.set_install_dir(version="2.2.4")
self.cluster.start()
if self.cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(self.query_node)
# Change system_auth keyspace replication factor to 2, otherwise replace will fail
session.execute("""ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy',
'replication_factor':2};""")
def _do_replace(self, same_address=False, jvm_option='replace_address',
wait_other_notice=False, wait_for_binary_proto=True,
replace_address=None, opts=None, data_center=None,
extra_jvm_args=None):
if replace_address is None:
replace_address = self.replaced_node.address()
# only create node if it's not yet created
if self.replacement_node is None:
replacement_address = '127.0.0.4'
if same_address:
replacement_address = self.replaced_node.address()
self.cluster.remove(self.replaced_node)
logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address))
self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042))
if opts is not None:
logger.debug("Setting options on replacement node: {}".format(opts))
self.replacement_node.set_configuration_options(opts)
self.cluster.add(self.replacement_node, False, data_center=data_center)
if extra_jvm_args is None:
extra_jvm_args = []
extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address),
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"])
self.replacement_node.start(jvm_args=extra_jvm_args,
wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice)
if self.cluster.cassandra_version() >= '2.2.8' and same_address:
self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement",
timeout=60)
def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE):
if self.replaced_node.is_running():
logger.debug("Stopping {}".format(self.replaced_node.name))
self.replaced_node.stop(gently=gently, wait_other_notice=True)
logger.debug("Testing node stoppage (query should fail).")
with pytest.raises((Unavailable, ReadTimeout)):
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {}'.format(table), consistency_level=cl)
session.execute(query)
def _insert_data(self, n='1k', rf=3, whitelist=False):
logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf))
self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf),
'-rate', 'threads=10'],
whitelist=whitelist)
self.cluster.flush()
time.sleep(20)
def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000):
logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit))
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl)
return rows_to_list(session.execute(query, timeout=20))
def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000,
restart_nodes=False):
assert len(initial_data) > 0, "Initial data must be greater than 0"
# query should work again
logger.debug("Stopping old nodes")
for node in self.cluster.nodelist():
if node.is_running() and node != self.replacement_node:
logger.debug("Stopping {}".format(node.name))
node.stop(gently=False, wait_other_notice=True)
logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit))
session = self.patient_exclusive_cql_connection(self.replacement_node)
assert_all(session, 'select * from {} LIMIT {}'.format(table, limit),
expected=initial_data,
cl=cl)
def _verify_replacement(self, node, same_address):
if not same_address:
if self.cluster.cassandra_version() >= '2.2.7':
address_prefix = '' if self.cluster.cassandra_version() >= '4.0' else '/'
node.watch_log_for("Node {}{} is replacing {}{}"
.format(address_prefix, self.replacement_node.address_for_current_version(),
address_prefix, self.replaced_node.address_for_current_version()),
timeout=60, filename='debug.log')
node.watch_log_for("Node {}{} will complete replacement of {}{} for tokens"
.format(address_prefix, self.replacement_node.address_for_current_version(),
address_prefix, self.replaced_node.address_for_current_version()), timeout=10)
node.watch_log_for("removing endpoint {}{}".format(address_prefix, self.replaced_node.address_for_current_version()),
timeout=60, filename='debug.log')
else:
node.watch_log_for("between /{} and /{}; /{} is the new owner"
.format(self.replaced_node.address(),
self.replacement_node.address(),
self.replacement_node.address()),
timeout=60)
def _verify_tokens_migrated_successfully(self, previous_log_size=None):
if not self.dtest_config.use_vnodes:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(self.replacement_node.get_conf_option('num_tokens'))
logger.debug("Verifying {} tokens migrated sucessfully".format(num_tokens))
logs = self.replacement_node.grep_log(r"Token (.*?) changing ownership from /{} to /{}"
.format(self.replaced_node.address(),
self.replacement_node.address()))
if (previous_log_size is not None):
assert len(logs) == previous_log_size
moved_tokens = set([l[1].group(1) for l in logs])
logger.debug("number of moved tokens: {}".format(len(moved_tokens)))
assert len(moved_tokens) == num_tokens
return len(logs)
def _test_insert_data_during_replace(self, same_address, mixed_versions=False):
"""
@jira_ticket CASSANDRA-8523
"""
default_install_dir = self.cluster.get_install_dir()
self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions)
self._insert_data(n='1k')
initial_data = self._fetch_initial_data()
self._stop_node_to_replace()
if mixed_versions:
logger.debug("Upgrading all except {} to current version".format(self.query_node.address()))
self.cluster.set_install_dir(install_dir=default_install_dir)
for node in self.cluster.nodelist():
if node.is_running() and node != self.query_node:
logger.debug("Upgrading {} to current version".format(node.address()))
node.stop(gently=True, wait_other_notice=True)
node.start(wait_other_notice=True, wait_for_binary_proto=True)
# start node in current version on write survey mode
self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"])
# Insert additional keys on query node
self._insert_data(n='2k', whitelist=True)
# If not same address or mixed versions, query node should forward writes to replacement node
# so we update initial data to reflect additional keys inserted during replace
if not same_address and not mixed_versions:
initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO)
logger.debug("Joining replaced node")
self.replacement_node.nodetool("join")
if not same_address:
for node in self.cluster.nodelist():
# if mixed version, query node is not upgraded so it will not print replacement log
if node.is_running() and (not mixed_versions or node != self.query_node):
self._verify_replacement(node, same_address)
self._verify_data(initial_data)
class BaseReplaceAddressTest(Tester):
@pytest.fixture(autouse=True)
def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
fixture_dtest_setup.ignore_log_patterns = (
# This one occurs when trying to send the migration to a
# node that hasn't started yet, and when it does, it gets
# replayed and everything is fine.
r'Can\'t send migration request: node.*is down',
r'Migration task failed to complete', # 10978
# ignore streaming error during bootstrap
r'Streaming error occurred',
r'failed stream session',
r'Failed to properly handshake with peer'
)
def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False):
logger.debug("Starting cluster with {} nodes.".format(n))
self.cluster.populate(n)
if opts is not None:
logger.debug("Setting cluster options: {}".format(opts))
self.cluster.set_configuration_options(opts)
self.cluster.set_batch_commitlog(enabled=True)
self.query_node = self.cluster.nodelist()[0]
self.replaced_node = self.cluster.nodelist()[-1]
self.cluster.seeds.remove(self.replaced_node)
if enable_byteman:
# set up byteman
self.query_node.byteman_port = '8100'
self.query_node.import_config_files()
if mixed_versions:
logger.debug("Starting nodes on version 2.2.4")
self.cluster.set_install_dir(version="2.2.4")
self.cluster.start()
if self.cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(self.query_node)
# Change system_auth keyspace replication factor to 2, otherwise replace will fail
session.execute("""ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy',
'replication_factor':2};""")
def _do_replace(self, same_address=False, jvm_option='replace_address',
wait_other_notice=False, wait_for_binary_proto=True,
replace_address=None, opts=None, data_center=None,
extra_jvm_args=None):
if replace_address is None:
replace_address = self.replaced_node.address()
# only create node if it's not yet created
if self.replacement_node is None:
replacement_address = '127.0.0.4'
if same_address:
replacement_address = self.replaced_node.address()
self.cluster.remove(self.replaced_node)
logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address))
self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042))
if opts is not None:
logger.debug("Setting options on replacement node: {}".format(opts))
self.replacement_node.set_configuration_options(opts)
self.cluster.add(self.replacement_node, False, data_center=data_center)
if extra_jvm_args is None:
extra_jvm_args = []
extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address),
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"])
self.replacement_node.start(jvm_args=extra_jvm_args,
wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice)
if self.cluster.cassandra_version() >= '2.2.8' and same_address:
self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement",
timeout=60)
def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE):
if self.replaced_node.is_running():
logger.debug("Stopping {}".format(self.replaced_node.name))
self.replaced_node.stop(gently=gently, wait_other_notice=True)
logger.debug("Testing node stoppage (query should fail).")
with pytest.raises((Unavailable, ReadTimeout)):
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {}'.format(table), consistency_level=cl)
session.execute(query)
def _insert_data(self, n='1k', rf=3, whitelist=False):
logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf))
self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf),
'-rate', 'threads=10'],
whitelist=whitelist)
self.cluster.flush()
time.sleep(20)
def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000):
logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit))
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl)
return rows_to_list(session.execute(query, timeout=20))
def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000,
restart_nodes=False):
assert len(initial_data) > 0, "Initial data must be greater than 0"
# query should work again
logger.debug("Stopping old nodes")
for node in self.cluster.nodelist():
if node.is_running() and node != self.replacement_node:
logger.debug("Stopping {}".format(node.name))
node.stop(gently=False, wait_other_notice=True)
logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit))
session = self.patient_exclusive_cql_connection(self.replacement_node)
assert_all(session, 'select * from {} LIMIT {}'.format(table, limit),
expected=initial_data,
cl=cl)
def _verify_replacement(self, node, same_address):
if not same_address:
if self.cluster.cassandra_version() >= '2.2.7':
node.watch_log_for("Node {} is replacing {}"
.format(self.replacement_node.address_for_current_version_slashy(),
self.replaced_node.address_for_current_version_slashy()),
timeout=60, filename='debug.log')
node.watch_log_for("Node {} will complete replacement of {} for tokens"
.format(self.replacement_node.address_for_current_version_slashy(),
self.replaced_node.address_for_current_version_slashy()), timeout=10)
node.watch_log_for("removing endpoint {}".format(self.replaced_node.address_for_current_version_slashy()),
timeout=60, filename='debug.log')
else:
node.watch_log_for("between /{} and /{}; /{} is the new owner"
.format(self.replaced_node.address(),
self.replacement_node.address(),
self.replacement_node.address()),
timeout=60)
def _verify_tokens_migrated_successfully(self, previous_log_size=None):
if not self.dtest_config.use_vnodes:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(self.replacement_node.get_conf_option('num_tokens'))
logger.debug("Verifying {} tokens migrated successfully".format(num_tokens))
replmnt_address = self.replacement_node.address_for_current_version_slashy()
repled_address = self.replaced_node.address_for_current_version_slashy()
token_ownership_log = r"Token (.*?) changing ownership from {} to {}".format(repled_address,
replmnt_address)
logs = self.replacement_node.grep_log(token_ownership_log)
if (previous_log_size is not None):
assert len(logs) == previous_log_size
moved_tokens = set([l[1].group(1) for l in logs])
logger.debug("number of moved tokens: {}".format(len(moved_tokens)))
assert len(moved_tokens) == num_tokens
return len(logs)
def _test_insert_data_during_replace(self, same_address, mixed_versions=False):
"""
@jira_ticket CASSANDRA-8523
"""
default_install_dir = self.cluster.get_install_dir()
self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions)
self._insert_data(n='1k')
initial_data = self._fetch_initial_data()
self._stop_node_to_replace()
if mixed_versions:
logger.debug("Upgrading all except {} to current version".format(self.query_node.address()))
self.cluster.set_install_dir(install_dir=default_install_dir)
for node in self.cluster.nodelist():
if node.is_running() and node != self.query_node:
logger.debug("Upgrading {} to current version".format(node.address()))
node.stop(gently=True, wait_other_notice=True)
node.start(wait_for_binary_proto=True)
# start node in current version on write survey mode
self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"])
# Insert additional keys on query node
self._insert_data(n='2k', whitelist=True)
# If not same address or mixed versions, query node should forward writes to replacement node
# so we update initial data to reflect additional keys inserted during replace
if not same_address and not mixed_versions:
initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO)
logger.debug("Joining replaced node")
self.replacement_node.nodetool("join")
if not same_address:
for node in self.cluster.nodelist():
# if mixed version, query node is not upgraded so it will not print replacement log
if node.is_running() and (not mixed_versions or node != self.query_node):
self._verify_replacement(node, same_address)
self._verify_data(initial_data)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress([
'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(
values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"],
wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
