def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
# try to replace an unassigned ip address
mark = node4.mark_log()
node4.start(replace_address='127.0.0.5', wait_other_notice=False)
node4.watch_log_for(
"java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip",
from_mark=mark)
assert_not_running(node4)
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
mark = node4.mark_log()
node4.start(replace_address='127.0.0.3', wait_other_notice=False)
node4.watch_log_for(
"java.lang.UnsupportedOperationException: Cannot replace a live node...",
from_mark=mark)
assert_not_running(node4)
def decommission_node_test(self):
debug("decommission_node_test()")
cluster = self.cluster
cluster.populate(1)
# create and add a new node, I must not be a seed, otherwise
# we get schema disagreement issues for awhile after decommissioning it.
node2 = Node('node2', cluster, True, ('127.0.0.2', 9160),
('127.0.0.2', 7000), '7200', '0', None)
cluster.add(node2, False)
[node1, node2] = cluster.nodelist()
node1.start()
node2.start()
wait(2)
cursor = self.cql_connection(node1).cursor()
self.prepare_for_changes(cursor)
node2.decommission()
wait(30)
self.validate_schema_consistent(node1)
self.make_schema_changes(cursor, namespace='ns1')
# create and add a new node
node3 = Node('node3', cluster, True, ('127.0.0.3', 9160),
('127.0.0.3', 7000), '7300', '0', None)
cluster.add(node3, True)
node3.start()
wait(30)
self.validate_schema_consistent(node1)
def test_upgrade_legacy_table(self):
"""
Upgrade with bringing up the legacy tables after the newer nodes (without legacy tables)
were started.
@jira_ticket CASSANDRA-12813
"""
cluster = self.cluster
# Forcing cluster version on purpose
cluster.set_install_dir(version="2.1.16")
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# Wait for default user to get created on one of the nodes
time.sleep(15)
# Upgrade to current version
for node in [node1, node2, node3]:
node.drain()
node.watch_log_for("DRAINED")
node.stop(gently=True)
self.set_node_to_current_version(node)
cluster.start()
# Make sure the system_auth table will get replicated to the node that we're going to replace
session = self.patient_cql_connection(node1, user='******', password='******')
session.execute("ALTER KEYSPACE system_auth WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 };")
cluster.repair()
cluster.stop()
# Replace the node
cluster.seeds.remove(node1)
cluster.remove(node1)
replacement_address = node1.address()
replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=(replacement_address, 9160), storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042))
self.set_node_to_current_version(replacement_node)
cluster.add(replacement_node, True)
replacement_node.start(wait_for_binary_proto=True)
node2.start(wait_for_binary_proto=True)
node3.start(wait_for_binary_proto=True)
replacement_node.watch_log_for('Initializing system_auth.credentials')
replacement_node.watch_log_for('Initializing system_auth.permissions')
replacement_node.watch_log_for('Initializing system_auth.users')
cluster.repair()
replacement_node.watch_log_for('Repair command')
# Should succeed. Will throw an NPE on pre-12813 code.
self.patient_cql_connection(replacement_node, user='******', password='******')
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
[node1, node2, node3] = cluster.nodelist()
debug("Inserting Data...")
if cluster.version() < "2.1":
node1.stress(
['-o', 'insert', '--num-keys=10000', '--replication-factor=3'])
else:
node1.stress(
['write', 'n=10000', '-schema', 'replication(factor=3)'])
cursor = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version(
) >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initialData = cursor.execute(query)
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160),
('127.0.0.4', 7000), '7400', '0', None,
('127.0.0.4', 9042))
cluster.add(node4, False)
#try to replace an unassigned ip address
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.5',
wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
def _init_new_loading_node(self, ks_name, create_stmt, use_thrift=False):
loading_node = Node(
name='node2',
cluster=self.cluster,
auto_bootstrap=False,
thrift_interface=('127.0.0.2', 9160) if use_thrift else None,
storage_interface=('127.0.0.2', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.2', 9042)
)
logger.debug('adding node')
self.cluster.add(loading_node, is_seed=True)
logger.debug('starting new node')
loading_node.start(wait_for_binary_proto=True)
logger.debug('recreating ks and table')
loading_session = self.patient_exclusive_cql_connection(loading_node)
create_ks(loading_session, ks_name, rf=1)
logger.debug('creating new table')
loading_session.execute(create_stmt)
logger.debug('stopping new node')
loading_session.cluster.shutdown()
loading_node.stop()
return loading_node
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
[node1,node2, node3] = cluster.nodelist()
debug("Inserting Data...")
if cluster.version() < "2.1":
node1.stress(['-o', 'insert', '--num-keys=10000', '--replication-factor=3'])
else:
node1.stress(['write', 'n=10000', '-schema', 'replication(factor=3)'])
cursor = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version() >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = cursor.execute(query)
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, ('127.0.0.4',9042))
cluster.add(node4, False)
#try to replace an unassigned ip address
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.5', wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
[node1,node2, node3] = cluster.nodelist()
debug("Inserting Data...")
if cluster.version() < "2.1":
node1.stress(['-o', 'insert', '--num-keys=10000', '--replication-factor=3'])
else:
node1.stress(['write', 'n=10000', '-schema', 'replication(factor=3)'])
cursor = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version() >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = cursor.execute(query)
#replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, ('127.0.0.4',9042))
cluster.add(node4, False)
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
checkError = node4.grep_log("java.lang.UnsupportedOperationException: Cannot replace a live node...")
self.assertEqual(len(checkError), 1)
def test_add_and_remove_node(self):
"""
Test that NEW_NODE and REMOVED_NODE are sent correctly as nodes join and leave.
@jira_ticket CASSANDRA-11038
"""
self.cluster.populate(1).start()
node1 = self.cluster.nodelist()[0]
waiter = NotificationWaiter(self, node1,
["STATUS_CHANGE", "TOPOLOGY_CHANGE"])
# need to block for up to 2 notifications (NEW_NODE and UP) so that these notifications
# don't confuse the state below
logger.debug("Waiting for unwanted notifications...")
waiter.wait_for_notifications(timeout=30, num_notifications=2)
waiter.clear_notifications()
session = self.patient_cql_connection(node1)
# reduce system_distributed RF to 2 so we don't require forceful decommission
session.execute(
"ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};"
)
session.execute(
"ALTER KEYSPACE system_traces WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};"
)
logger.debug("Adding second node...")
node2 = Node('node2',
self.cluster,
True,
None, ('127.0.0.2', 7000),
'7200',
'0',
None,
binary_interface=('127.0.0.2', 9042))
self.cluster.add(node2, False)
node2.start()
logger.debug("Waiting for notifications from {}".format(
waiter.address))
notifications = waiter.wait_for_notifications(timeout=120.0,
num_notifications=2)
assert 2 == len(notifications), notifications
for notification in notifications:
assert get_ip_from_node(node2) == notification["address"][0]
assert "NEW_NODE" == notifications[0]["change_type"]
assert "UP" == notifications[1]["change_type"]
logger.debug("Removing second node...")
waiter.clear_notifications()
node2.decommission()
node2.stop(gently=False)
logger.debug("Waiting for notifications from {}".format(
waiter.address))
notifications = waiter.wait_for_notifications(timeout=120.0,
num_notifications=2)
assert 2 == len(notifications), notifications
for notification in notifications:
assert get_ip_from_node(node2) == notification["address"][0]
assert "REMOVED_NODE" == notifications[0]["change_type"]
assert "DOWN" == notifications[1]["change_type"]
def _test_disk_balance_replace(self, same_address):
logger.debug("Creating cluster")
cluster = self.cluster
if self.dtest_config.use_vnodes:
cluster.set_configuration_options(values={'num_tokens': 256})
# apparently we have legitimate errors in the log when bootstrapping (see bootstrap_test.py)
self.fixture_dtest_setup.allow_log_errors = True
cluster.populate(4).start(wait_for_binary_proto=True)
node1 = cluster.nodes['node1']
logger.debug("Populating")
node1.stress(['write', 'n=50k', 'no-warmup', '-rate', 'threads=100', '-schema', 'replication(factor=3)', 'compaction(strategy=SizeTieredCompactionStrategy,enabled=false)'])
cluster.flush()
logger.debug("Stopping and removing node2")
node2 = cluster.nodes['node2']
node2.stop(gently=False)
self.cluster.remove(node2)
node5_address = node2.address() if same_address else '127.0.0.5'
logger.debug("Starting replacement node")
node5 = Node('node5', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(node5_address, 7000),
jmx_port='7500', remote_debug_port='0', initial_token=None,
binary_interface=(node5_address, 9042))
self.cluster.add(node5, False)
node5.start(jvm_args=["-Dcassandra.replace_address_first_boot={}".format(node2.address())],
wait_for_binary_proto=True,
wait_other_notice=True)
logger.debug("Checking replacement node is balanced")
self.assert_balanced(node5)
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def decommission_node_test(self):
debug("decommission_node_test()")
cluster = self.cluster
cluster.populate(1)
# create and add a new node, I must not be a seed, otherwise
# we get schema disagreement issues for awhile after decommissioning it.
node2 = Node("node2", cluster, True, ("127.0.0.2", 9160), ("127.0.0.2", 7000), "7200", None)
cluster.add(node2, False)
[node1, node2] = cluster.nodelist()
node1.start()
node2.start()
wait(2)
cursor = self.cql_connection(node1).cursor()
self.prepare_for_changes(cursor)
node2.decommission()
wait(30)
self.validate_schema_consistent(node1)
self.make_schema_changes(cursor, namespace="ns1")
# create and add a new node
node3 = Node("node3", cluster, True, ("127.0.0.3", 9160), ("127.0.0.3", 7000), "7300", None)
cluster.add(node3, True)
node3.start()
wait(30)
self.validate_schema_consistent(node1)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def multiple_repair_test(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 3)
self.create_cf(session, 'cf', read_repair=0.0, columns={'c1': 'text', 'c2': 'text'})
debug("insert data")
insert_c1c2(session, keys=range(1, 50), consistency=ConsistencyLevel.ALL)
node1.flush()
debug("bringing down node 3")
node3.flush()
node3.stop(gently=False)
debug("inserting additional data into node 1 and 2")
insert_c1c2(session, keys=range(50, 100), consistency=ConsistencyLevel.TWO)
node1.flush()
node2.flush()
debug("restarting and repairing node 3")
node3.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node3.repair()
else:
node3.nodetool("repair -par -inc")
# wait stream handlers to be closed on windows
# after session is finished (See CASSANDRA-10644)
if is_win:
time.sleep(2)
debug("stopping node 2")
node2.stop(gently=False)
debug("inserting data in nodes 1 and 3")
insert_c1c2(session, keys=range(100, 150), consistency=ConsistencyLevel.TWO)
node1.flush()
node3.flush()
debug("start and repair node 2")
node2.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node2.repair()
else:
node2.nodetool("repair -par -inc")
debug("replace node and check data integrity")
node3.stop(gently=False)
node5 = Node('node5', cluster, True, ('127.0.0.5', 9160), ('127.0.0.5', 7000), '7500', '0', None, ('127.0.0.5', 9042))
cluster.add(node5, False)
node5.start(replace_address='127.0.0.3', wait_other_notice=True)
assert_one(session, "SELECT COUNT(*) FROM ks.cf LIMIT 200", [149])
def resumable_replace_test(self):
"""Test resumable bootstrap while replacing node"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster,
True, ('127.0.0.4', 9160), ('127.0.0.4', 7000),
'7400',
'0',
None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"])
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = list(
session.execute(
"SELECT bootstrapped FROM system.local WHERE key='local'"))
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
def test_rf_gt_nodes_multidc_should_succeed(self):
"""
Validating a KS with RF > N on multi DC doesn't break bootstrap
@jira_ticket CASSANDRA-16296 CASSANDRA-16411
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate([1, 1])
cluster.start()
node1 = cluster.nodelist()[0]
node2 = cluster.nodelist()[1]
session = self.patient_exclusive_cql_connection(node1)
session.execute(
"CREATE KEYSPACE k WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'dc1' : '3'}"
)
if cluster.version() >= '4.0':
warning = 'Your replication factor 3 for keyspace k is higher than the number of nodes 1 for datacenter dc1'
assert len(node1.grep_log(warning)) == 1
assert len(node2.grep_log(warning)) == 0
session.execute(
"ALTER KEYSPACE k WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'dc1' : '2'}"
)
session.execute(
"CREATE TABLE k.testgtrfmultidc (KEY text PRIMARY KEY)")
session.execute(
"INSERT INTO k.testgtrfmultidc (KEY) VALUES ('test_rf_gt_nodes_multidc_should_succeed')"
)
if cluster.version() >= '4.0':
warning = 'Your replication factor 2 for keyspace k is higher than the number of nodes 1 for datacenter dc1'
assert len(node1.grep_log(warning)) == 1
assert len(node2.grep_log(warning)) == 0
marks = map(lambda n: n.mark_log(), cluster.nodelist())
node3 = Node(name='node3',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.3', 9160),
storage_interface=('127.0.0.3', 7000),
jmx_port='7300',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.3', 9042))
cluster.add(node3, is_seed=False, data_center="dc1")
node3.start(wait_for_binary_proto=True)
if cluster.version() >= '4.0':
warning = 'is higher than the number of nodes'
for (node, mark) in zip(cluster.nodelist(), marks):
assert len(node.grep_log(warning, from_mark=mark)) == 0
session3 = self.patient_exclusive_cql_connection(node3)
assert_one(session3, "SELECT * FROM k.testgtrfmultidc",
["test_rf_gt_nodes_multidc_should_succeed"])
def multiple_repair_test(self):
cluster = self.cluster
cluster.populate(3).start()
[node1, node2, node3] = cluster.nodelist()
cursor = self.patient_cql_connection(node1)
self.create_ks(cursor, 'ks', 3)
self.create_cf(cursor, 'cf', read_repair=0.0, columns={'c1': 'text', 'c2': 'text'})
debug("insert data")
for x in range(1, 50):
insert_c1c2(cursor, x, ConsistencyLevel.ALL)
node1.flush()
debug("bringing down node 3")
node3.flush()
node3.stop(gently=False)
debug("inserting additional data into node 1 and 2")
for y in range(50, 100):
insert_c1c2(cursor, y, ConsistencyLevel.TWO)
node1.flush()
node2.flush()
debug("restarting and repairing node 3")
node3.start()
if cluster.version() >= "3.0":
node3.repair()
else:
node3.nodetool("repair -par -inc")
debug("stopping node 2")
node2.stop(gently=False)
debug("inserting data in nodes 1 and 3")
for z in range(100, 150):
insert_c1c2(cursor, z, ConsistencyLevel.TWO)
node1.flush()
node3.flush()
debug("start and repair node 2")
node2.start()
if cluster.version() >= "3.0":
node2.repair()
else:
node2.nodetool("repair -par -inc")
debug("replace node and check data integrity")
node3.stop(gently=False)
node5 = Node('node5', cluster, True, ('127.0.0.5', 9160), ('127.0.0.5', 7000), '7500', '0', None, ('127.0.0.5',9042))
cluster.add(node5, False)
node5.start(replace_address = '127.0.0.3', wait_other_notice=True)
assert_one(cursor, "SELECT COUNT(*) FROM ks.cf LIMIT 200", [149])
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = list(session.execute("SELECT bootstrapped FROM system.local WHERE key='local'"))
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = list(session.execute("SELECT bootstrapped FROM system.local WHERE key='local'"))
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
def replace_with_insufficient_replicas_test(self):
"""
Test that replace fails when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress([
'write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=2)'
])
# stop node to replace
debug("Stopping node to replace.")
node3.stop(wait_other_notice=True)
# stop other replica
debug("Stopping node2 (other replica)")
node2.stop(wait_other_notice=True)
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3',
wait_for_binary_proto=False,
wait_other_notice=False)
# replace should fail due to insufficient replicas
node4.watch_log_for(
"Unable to find sufficient sources for streaming range")
assert_not_running(node4)
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
# try to replace an unassigned ip address
mark = node4.mark_log()
node4.start(replace_address='127.0.0.5', wait_other_notice=False)
node4.watch_log_for("java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip", from_mark=mark)
self.check_not_running(node4)
def issue_150_test(self):
self.cluster = Cluster(CLUSTER_PATH, "150", cassandra_version='2.0.9')
self.cluster.populate([1, 2], use_vnodes=True)
self.cluster.start()
dcs = [node.data_center for node in self.cluster.nodelist()]
dcs.append('dc2')
node4 = Node('node4', self.cluster, True, ('127.0.0.4', 9160),
('127.0.0.4', 7000), '7400', '2000', None)
self.cluster.add(node4, False, 'dc2')
node4.start()
dcs_2 = [node.data_center for node in self.cluster.nodelist()]
self.assertListEqual(dcs, dcs_2)
node4.nodetool('status')
def issue_150_test(self):
self.cluster = Cluster(CLUSTER_PATH, "150", cassandra_version='2.0.9')
self.cluster.populate([1, 2], use_vnodes=True)
self.cluster.start()
dcs = [node.data_center for node in self.cluster.nodelist()]
dcs.append('dc2')
node4 = Node('node4', self.cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000),
'7400', '2000', None)
self.cluster.add(node4, False, 'dc2')
node4.start()
dcs_2 = [node.data_center for node in self.cluster.nodelist()]
self.assertItemsEqual(dcs, dcs_2)
node4.nodetool('status')
def test_decommission_node(self):
logger.debug("decommission_node_test()")
cluster = self.cluster
cluster.populate(1)
# create and add a new node, I must not be a seed, otherwise
# we get schema disagreement issues for awhile after decommissioning it.
node2 = Node('node2',
cluster,
True,
('127.0.0.2', 9160),
('127.0.0.2', 7000),
'7200',
'0',
None,
binary_interface=('127.0.0.2', 9042))
cluster.add(node2, False)
node1, node2 = cluster.nodelist()
node1.start(wait_for_binary_proto=True)
node2.start(wait_for_binary_proto=True)
wait(2)
session = self.patient_cql_connection(node1)
self.prepare_for_changes(session)
node2.decommission()
wait(30)
self.validate_schema_consistent(node1)
self.make_schema_changes(session, namespace='ns1')
# create and add a new node
node3 = Node('node3',
cluster,
True,
('127.0.0.3', 9160),
('127.0.0.3', 7000),
'7300',
'0',
None,
binary_interface=('127.0.0.3', 9042))
cluster.add(node3, True)
node3.start(wait_for_binary_proto=True)
wait(30)
self.validate_schema_consistent(node1)
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, ('127.0.0.4', 9042))
cluster.add(node4, False)
#try to replace an unassigned ip address
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.5', wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
mark = node4.mark_log()
node4.start(replace_address='127.0.0.3', wait_other_notice=False)
node4.watch_log_for("java.lang.UnsupportedOperationException: Cannot replace a live node...", from_mark=mark)
self.check_not_running(node4)
def resumable_replace_test(self):
"""Test resumable bootstrap while replacing node"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version() >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = session.execute(query)
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, ('127.0.0.4', 9042))
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"])
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
session = self.exclusive_cql_connection(node4)
rows = session.execute("SELECT bootstrapped FROM system.local WHERE key='local'")
assert len(rows) == 1
assert rows[0][0] == 'COMPLETED', rows[0][0]
#query should work again
debug("Verifying querying works again.")
finalData = session.execute(query)
self.assertListEqual(initialData, finalData)
def add_and_remove_node_test(self):
"""
Test that NEW_NODE and REMOVED_NODE are sent correctly as nodes join and leave.
@jira_ticket CASSANDRA-11038
"""
self.cluster.populate(1).start(wait_for_binary_proto=True)
node1 = self.cluster.nodelist()[0]
waiter = NotificationWaiter(self, node1,
["STATUS_CHANGE", "TOPOLOGY_CHANGE"])
# need to block for up to 2 notifications (NEW_NODE and UP) so that these notifications
# don't confuse the state below
debug("Waiting for unwanted notifications...")
waiter.wait_for_notifications(timeout=30, num_notifications=2)
waiter.clear_notifications()
debug("Adding second node...")
node2 = Node('node2', self.cluster, True, ('127.0.0.2', 9160),
('127.0.0.2', 7000), '7200', '0', None,
('127.0.0.2', 9042))
self.cluster.add(node2, False)
node2.start(wait_other_notice=True)
debug("Waiting for notifications from {}".format(waiter.address))
notifications = waiter.wait_for_notifications(timeout=60.0,
num_notifications=2)
self.assertEquals(2, len(notifications), notifications)
for notification in notifications:
self.assertEquals(self.get_ip_from_node(node2),
notification["address"][0])
self.assertEquals("NEW_NODE", notifications[0]["change_type"])
self.assertEquals("UP", notifications[1]["change_type"])
debug("Removing second node...")
waiter.clear_notifications()
node2.decommission()
node2.stop(gently=False)
debug("Waiting for notifications from {}".format(waiter.address))
notifications = waiter.wait_for_notifications(timeout=60.0,
num_notifications=2)
self.assertEquals(2, len(notifications), notifications)
for notification in notifications:
self.assertEquals(self.get_ip_from_node(node2),
notification["address"][0])
self.assertEquals("REMOVED_NODE", notifications[0]["change_type"])
self.assertEquals("DOWN", notifications[1]["change_type"])
def replace_nonexistent_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
debug('Start node 4 and replace an address with no node')
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
# try to replace an unassigned ip address
mark = node4.mark_log()
node4.start(replace_address='127.0.0.5', wait_other_notice=False)
node4.watch_log_for("java.lang.RuntimeException: Cannot replace_address /127.0.0.5 because it doesn't exist in gossip", from_mark=mark)
assert_not_running(node4)
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
mark = node4.mark_log()
node4.start(replace_address='127.0.0.3', wait_other_notice=False)
node4.watch_log_for("java.lang.UnsupportedOperationException: Cannot replace a live node...", from_mark=mark)
assert_not_running(node4)
def test_add_and_remove_node(self):
"""
Test that NEW_NODE and REMOVED_NODE are sent correctly as nodes join and leave.
@jira_ticket CASSANDRA-11038
"""
self.cluster.populate(1).start(wait_for_binary_proto=True)
node1 = self.cluster.nodelist()[0]
waiter = NotificationWaiter(self, node1, ["STATUS_CHANGE", "TOPOLOGY_CHANGE"])
# need to block for up to 2 notifications (NEW_NODE and UP) so that these notifications
# don't confuse the state below
logger.debug("Waiting for unwanted notifications...")
waiter.wait_for_notifications(timeout=30, num_notifications=2)
waiter.clear_notifications()
session = self.patient_cql_connection(node1)
# reduce system_distributed RF to 2 so we don't require forceful decommission
session.execute("ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};")
session.execute("ALTER KEYSPACE system_traces WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};")
logger.debug("Adding second node...")
node2 = Node('node2', self.cluster, True, None, ('127.0.0.2', 7000), '7200', '0', None, binary_interface=('127.0.0.2', 9042))
self.cluster.add(node2, False)
node2.start(wait_other_notice=True)
logger.debug("Waiting for notifications from {}".format(waiter.address))
notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=2)
assert 2 == len(notifications), notifications
for notification in notifications:
assert get_ip_from_node(node2) == notification["address"][0]
assert "NEW_NODE" == notifications[0]["change_type"]
assert "UP" == notifications[1]["change_type"]
logger.debug("Removing second node...")
waiter.clear_notifications()
node2.decommission()
node2.stop(gently=False)
logger.debug("Waiting for notifications from {}".format(waiter.address))
notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=2)
assert 2 == len(notifications), notifications
for notification in notifications:
assert get_ip_from_node(node2) == notification["address"][0]
assert "REMOVED_NODE" == notifications[0]["change_type"]
assert "DOWN" == notifications[1]["change_type"]
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
#replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, ('127.0.0.4', 9042))
cluster.add(node4, False)
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
checkError = node4.grep_log("java.lang.UnsupportedOperationException: Cannot replace a live node...")
self.assertEqual(len(checkError), 1)
def replace_with_insufficient_replicas_test(self):
"""
Test that replace fails when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress(['write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=2)'])
# stop node to replace
debug("Stopping node to replace.")
node3.stop(wait_other_notice=True)
# stop other replica
debug("Stopping node2 (other replica)")
node2.stop(wait_other_notice=True)
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=False, wait_other_notice=False)
# replace should fail due to insufficient replicas
node4.watch_log_for("Unable to find sufficient sources for streaming range")
assert_not_running(node4)
def add_and_remove_node_test(self):
"""
Test that NEW_NODE and REMOVED_NODE are sent correctly as nodes join and leave.
@jira_ticket CASSANDRA-11038
"""
self.cluster.populate(1).start(wait_for_binary_proto=True)
node1 = self.cluster.nodelist()[0]
waiter = NotificationWaiter(self, node1, ["STATUS_CHANGE", "TOPOLOGY_CHANGE"])
# need to block for up to 2 notifications (NEW_NODE and UP) so that these notifications
# don't confuse the state below
debug("Waiting for unwanted notifications...")
waiter.wait_for_notifications(timeout=30, num_notifications=2)
waiter.clear_notifications()
debug("Adding second node...")
node2 = Node('node2', self.cluster, True, None, ('127.0.0.2', 7000), '7200', '0', None, binary_interface=('127.0.0.2', 9042))
self.cluster.add(node2, False)
node2.start(wait_other_notice=True)
debug("Waiting for notifications from {}".format(waiter.address))
notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=2)
self.assertEquals(2, len(notifications), notifications)
for notification in notifications:
self.assertEquals(get_ip_from_node(node2), notification["address"][0])
self.assertEquals("NEW_NODE", notifications[0]["change_type"])
self.assertEquals("UP", notifications[1]["change_type"])
debug("Removing second node...")
waiter.clear_notifications()
node2.decommission()
node2.stop(gently=False)
debug("Waiting for notifications from {}".format(waiter.address))
notifications = waiter.wait_for_notifications(timeout=60.0, num_notifications=2)
self.assertEquals(2, len(notifications), notifications)
for notification in notifications:
self.assertEquals(get_ip_from_node(node2), notification["address"][0])
self.assertEquals("REMOVED_NODE", notifications[0]["change_type"])
self.assertEquals("DOWN", notifications[1]["change_type"])
def replace_active_node_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
[node1, node2, node3] = cluster.nodelist()
debug("Inserting Data...")
if cluster.version() < "2.1":
node1.stress(
['-o', 'insert', '--num-keys=10000', '--replication-factor=3'])
else:
node1.stress(
['write', 'n=10000', '-schema', 'replication(factor=3)'])
cursor = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1' if self.cluster.version(
) >= '2.1' else '"Keyspace1"."Standard1"'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initialData = cursor.execute(query)
#replace active node 3 with node 4
debug("Starting node 4 to replace active node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160),
('127.0.0.4', 7000), '7400', '0', None,
('127.0.0.4', 9042))
cluster.add(node4, False)
with self.assertRaises(NodeError):
try:
node4.start(replace_address='127.0.0.3',
wait_for_binary_proto=True)
except (NodeError, TimeoutError):
raise NodeError("Node could not start.")
checkError = node4.grep_log(
"java.lang.UnsupportedOperationException: Cannot replace a live node..."
)
self.assertEqual(len(checkError), 1)
def decommission_node_schema_check_test(self):
cluster = self.cluster
cluster.populate(1)
# create and add a non-seed node.
node2 = Node('node2',
cluster,
True,
('127.0.0.2', 9160),
('127.0.0.2', 7000),
'7200',
None)
cluster.add(node2, False)
[node1, node2] = cluster.nodelist()
node1.start()
node2.start()
time.sleep(2)
node2.decommission()
time.sleep(30)
self.validate_schema_consistent(node1)
class BaseReplaceAddressTest(Tester):
@pytest.fixture(autouse=True)
def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
fixture_dtest_setup.ignore_log_patterns = (
# This one occurs when trying to send the migration to a
# node that hasn't started yet, and when it does, it gets
# replayed and everything is fine.
r'Can\'t send migration request: node.*is down',
r'Migration task failed to complete', # 10978
# ignore streaming error during bootstrap
r'Streaming error occurred',
r'failed stream session',
r'Failed to properly handshake with peer'
)
def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False):
logger.debug("Starting cluster with {} nodes.".format(n))
self.cluster.populate(n)
if opts is not None:
logger.debug("Setting cluster options: {}".format(opts))
self.cluster.set_configuration_options(opts)
self.cluster.set_batch_commitlog(enabled=True)
self.query_node = self.cluster.nodelist()[0]
self.replaced_node = self.cluster.nodelist()[-1]
self.cluster.seeds.remove(self.replaced_node)
if enable_byteman:
# set up byteman
self.query_node.byteman_port = '8100'
self.query_node.import_config_files()
if mixed_versions:
logger.debug("Starting nodes on version 2.2.4")
self.cluster.set_install_dir(version="2.2.4")
self.cluster.start()
if self.cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(self.query_node)
# Change system_auth keyspace replication factor to 2, otherwise replace will fail
session.execute("""ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy',
'replication_factor':2};""")
def _do_replace(self, same_address=False, jvm_option='replace_address',
wait_other_notice=False, wait_for_binary_proto=True,
replace_address=None, opts=None, data_center=None,
extra_jvm_args=None):
if replace_address is None:
replace_address = self.replaced_node.address()
# only create node if it's not yet created
if self.replacement_node is None:
replacement_address = '127.0.0.4'
if same_address:
replacement_address = self.replaced_node.address()
self.cluster.remove(self.replaced_node)
logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address))
self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042))
if opts is not None:
logger.debug("Setting options on replacement node: {}".format(opts))
self.replacement_node.set_configuration_options(opts)
self.cluster.add(self.replacement_node, False, data_center=data_center)
if extra_jvm_args is None:
extra_jvm_args = []
extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address),
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"])
self.replacement_node.start(jvm_args=extra_jvm_args,
wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice)
if self.cluster.cassandra_version() >= '2.2.8' and same_address:
self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement",
timeout=60)
def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE):
if self.replaced_node.is_running():
logger.debug("Stopping {}".format(self.replaced_node.name))
self.replaced_node.stop(gently=gently, wait_other_notice=True)
logger.debug("Testing node stoppage (query should fail).")
with pytest.raises((Unavailable, ReadTimeout)):
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {}'.format(table), consistency_level=cl)
session.execute(query)
def _insert_data(self, n='1k', rf=3, whitelist=False):
logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf))
self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf),
'-rate', 'threads=10'],
whitelist=whitelist)
self.cluster.flush()
time.sleep(20)
def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000):
logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit))
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl)
return rows_to_list(session.execute(query, timeout=20))
def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000,
restart_nodes=False):
assert len(initial_data) > 0, "Initial data must be greater than 0"
# query should work again
logger.debug("Stopping old nodes")
for node in self.cluster.nodelist():
if node.is_running() and node != self.replacement_node:
logger.debug("Stopping {}".format(node.name))
node.stop(gently=False, wait_other_notice=True)
logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit))
session = self.patient_exclusive_cql_connection(self.replacement_node)
assert_all(session, 'select * from {} LIMIT {}'.format(table, limit),
expected=initial_data,
cl=cl)
def _verify_replacement(self, node, same_address):
if not same_address:
if self.cluster.cassandra_version() >= '2.2.7':
node.watch_log_for("Node {} is replacing {}"
.format(self.replacement_node.address_for_current_version_slashy(),
self.replaced_node.address_for_current_version_slashy()),
timeout=60, filename='debug.log')
node.watch_log_for("Node {} will complete replacement of {} for tokens"
.format(self.replacement_node.address_for_current_version_slashy(),
self.replaced_node.address_for_current_version_slashy()), timeout=10)
node.watch_log_for("removing endpoint {}".format(self.replaced_node.address_for_current_version_slashy()),
timeout=60, filename='debug.log')
else:
node.watch_log_for("between /{} and /{}; /{} is the new owner"
.format(self.replaced_node.address(),
self.replacement_node.address(),
self.replacement_node.address()),
timeout=60)
def _verify_tokens_migrated_successfully(self, previous_log_size=None):
if not self.dtest_config.use_vnodes:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(self.replacement_node.get_conf_option('num_tokens'))
logger.debug("Verifying {} tokens migrated successfully".format(num_tokens))
replmnt_address = self.replacement_node.address_for_current_version_slashy()
repled_address = self.replaced_node.address_for_current_version_slashy()
token_ownership_log = r"Token (.*?) changing ownership from {} to {}".format(repled_address,
replmnt_address)
logs = self.replacement_node.grep_log(token_ownership_log)
if (previous_log_size is not None):
assert len(logs) == previous_log_size
moved_tokens = set([l[1].group(1) for l in logs])
logger.debug("number of moved tokens: {}".format(len(moved_tokens)))
assert len(moved_tokens) == num_tokens
return len(logs)
def _test_insert_data_during_replace(self, same_address, mixed_versions=False):
"""
@jira_ticket CASSANDRA-8523
"""
default_install_dir = self.cluster.get_install_dir()
self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions)
self._insert_data(n='1k')
initial_data = self._fetch_initial_data()
self._stop_node_to_replace()
if mixed_versions:
logger.debug("Upgrading all except {} to current version".format(self.query_node.address()))
self.cluster.set_install_dir(install_dir=default_install_dir)
for node in self.cluster.nodelist():
if node.is_running() and node != self.query_node:
logger.debug("Upgrading {} to current version".format(node.address()))
node.stop(gently=True, wait_other_notice=True)
node.start(wait_for_binary_proto=True)
# start node in current version on write survey mode
self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"])
# Insert additional keys on query node
self._insert_data(n='2k', whitelist=True)
# If not same address or mixed versions, query node should forward writes to replacement node
# so we update initial data to reflect additional keys inserted during replace
if not same_address and not mixed_versions:
initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO)
logger.debug("Joining replaced node")
self.replacement_node.nodetool("join")
if not same_address:
for node in self.cluster.nodelist():
# if mixed version, query node is not upgraded so it will not print replacement log
if node.is_running() and (not mixed_versions or node != self.query_node):
self._verify_replacement(node, same_address)
self._verify_data(initial_data)
def _replace_node_test(self, gently):
"""
Check that the replace address function correctly replaces a node that has failed in a cluster.
Create a cluster, cause a node to fail, and bring up a new node with the replace_address parameter.
Check that tokens are migrated and that data is replicated properly.
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
numNodes = 1
else:
# a little hacky but grep_log returns the whole line...
numNodes = int(node3.get_conf_option('num_tokens'))
debug(numNodes)
debug("Inserting Data...")
node1.stress(['write', 'n=10K', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
session.default_timeout = 45
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
# stop node, query should not work with consistency 3
debug("Stopping node 3.")
node3.stop(gently=gently, wait_other_notice=True)
debug("Testing node stoppage (query should fail).")
with self.assertRaises(NodeUnavailable):
try:
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
session.execute(query)
except (Unavailable, ReadTimeout):
raise NodeUnavailable("Node could not be queried.")
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=True)
# query should work again
debug("Verifying querying works again.")
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
debug("Verifying tokens migrated sucessfully")
movedTokensList = node4.grep_log("Token .* changing ownership from /127.0.0.3 to /127.0.0.4")
debug(movedTokensList[0])
self.assertEqual(len(movedTokensList), numNodes)
# check that restarting node 3 doesn't work
debug("Try to restart node 3 (should fail)")
node3.start(wait_other_notice=False)
checkCollision = node1.grep_log("between /127.0.0.3 and /127.0.0.4; /127.0.0.4 is the new owner")
debug(checkCollision)
self.assertEqual(len(checkCollision), 1)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress([
'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(
values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"],
wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress([
'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(
values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(
jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"],
wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh",
from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def replace_first_boot_test(self):
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
numNodes = 1
else:
# a little hacky but grep_log returns the whole line...
numNodes = int(node3.get_conf_option('num_tokens'))
debug(numNodes)
debug("Inserting Data...")
node1.stress(['write', 'n=10K', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initialData = list(session.execute(query))
# stop node, query should not work with consistency 3
debug("Stopping node 3.")
node3.stop(gently=False)
debug("Testing node stoppage (query should fail).")
with self.assertRaises(NodeUnavailable):
try:
session.execute(query, timeout=30)
except (Unavailable, ReadTimeout):
raise NodeUnavailable("Node could not be queried.")
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_for_binary_proto=True)
# query should work again
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
debug("Verifying tokens migrated sucessfully")
movedTokensList = node4.grep_log("Token .* changing ownership from /127.0.0.3 to /127.0.0.4")
debug(movedTokensList[0])
self.assertEqual(len(movedTokensList), numNodes)
# check that restarting node 3 doesn't work
debug("Try to restart node 3 (should fail)")
node3.start(wait_other_notice=False)
checkCollision = node1.grep_log("between /127.0.0.3 and /127.0.0.4; /127.0.0.4 is the new owner")
debug(checkCollision)
self.assertEqual(len(checkCollision), 1)
# restart node4 (if error's might have to change num_tokens)
node4.stop(gently=False)
node4.start(wait_for_binary_proto=True, wait_other_notice=False)
debug("Verifying querying works again.")
finalData = list(session.execute(query))
self.assertListEqual(initialData, finalData)
# we redo this check because restarting node should not result in tokens being moved again, ie number should be same
debug("Verifying tokens migrated sucessfully")
movedTokensList = node4.grep_log("Token .* changing ownership from /127.0.0.3 to /127.0.0.4")
debug(movedTokensList[0])
self.assertEqual(len(movedTokensList), numNodes)
def _replace_node_test(self, gently):
"""
Check that the replace address function correctly replaces a node that has failed in a cluster.
Create a cluster, cause a node to fail, and bring up a new node with the replace_address parameter.
Check that tokens are migrated and that data is replicated properly.
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress([
'write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=3)'
])
session = self.patient_cql_connection(node1)
session.default_timeout = 45
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
# stop node, query should not work with consistency 3
debug("Stopping node 3.")
node3.stop(gently=gently, wait_other_notice=True)
debug("Testing node stoppage (query should fail).")
with self.assertRaises(NodeUnavailable):
try:
query = SimpleStatement(
'select * from %s LIMIT 1' % stress_table,
consistency_level=ConsistencyLevel.THREE)
session.execute(query)
except (Unavailable, ReadTimeout):
raise NodeUnavailable("Node could not be queried.")
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=True)
debug("Verifying tokens migrated sucessfully")
moved_tokens = node4.grep_log(
"Token .* changing ownership from /127.0.0.3 to /127.0.0.4")
debug("number of moved tokens: {}".format(len(moved_tokens)))
self.assertEqual(len(moved_tokens), num_tokens)
# check that restarting node 3 doesn't work
debug("Try to restart node 3 (should fail)")
node3.start(wait_other_notice=False)
collision_log = node1.grep_log(
"between /127.0.0.3 and /127.0.0.4; /127.0.0.4 is the new owner")
debug(collision_log)
self.assertEqual(len(collision_log), 1)
node3.stop(gently=False)
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session,
'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def multiple_repair_test(self):
"""
* Launch a three node cluster
* Create a keyspace with RF 3 and a table
* Insert 49 rows
* Stop node3
* Insert 50 more rows
* Restart node3
* Issue an incremental repair on node3
* Stop node2
* Insert a final50 rows
* Restart node2
* Issue an incremental repair on node2
* Replace node3 with a new node
* Verify data integrity
# TODO: Several more verifications of data need to be interspersed throughout the test. The final assertion is insufficient.
@jira_ticket CASSANDRA-10644
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node1)
create_ks(session, 'ks', 3)
create_cf(session, 'cf', read_repair=0.0, columns={'c1': 'text', 'c2': 'text'})
debug("insert data")
insert_c1c2(session, keys=range(1, 50), consistency=ConsistencyLevel.ALL)
node1.flush()
debug("bringing down node 3")
node3.flush()
node3.stop(gently=False)
debug("inserting additional data into node 1 and 2")
insert_c1c2(session, keys=range(50, 100), consistency=ConsistencyLevel.TWO)
node1.flush()
node2.flush()
debug("restarting and repairing node 3")
node3.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node3.repair()
else:
node3.nodetool("repair -par -inc")
# wait stream handlers to be closed on windows
# after session is finished (See CASSANDRA-10644)
if is_win:
time.sleep(2)
debug("stopping node 2")
node2.stop(gently=False)
debug("inserting data in nodes 1 and 3")
insert_c1c2(session, keys=range(100, 150), consistency=ConsistencyLevel.TWO)
node1.flush()
node3.flush()
debug("start and repair node 2")
node2.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node2.repair()
else:
node2.nodetool("repair -par -inc")
debug("replace node and check data integrity")
node3.stop(gently=False)
node5 = Node('node5', cluster, True, ('127.0.0.5', 9160), ('127.0.0.5', 7000), '7500', '0', None, ('127.0.0.5', 9042))
cluster.add(node5, False)
node5.start(replace_address='127.0.0.3', wait_other_notice=True)
assert_one(session, "SELECT COUNT(*) FROM ks.cf LIMIT 200", [149])
def multiple_repair_test(self):
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 3)
self.create_cf(session,
'cf',
read_repair=0.0,
columns={
'c1': 'text',
'c2': 'text'
})
debug("insert data")
insert_c1c2(session,
keys=range(1, 50),
consistency=ConsistencyLevel.ALL)
node1.flush()
debug("bringing down node 3")
node3.flush()
node3.stop(gently=False)
debug("inserting additional data into node 1 and 2")
insert_c1c2(session,
keys=range(50, 100),
consistency=ConsistencyLevel.TWO)
node1.flush()
node2.flush()
debug("restarting and repairing node 3")
node3.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node3.repair()
else:
node3.nodetool("repair -par -inc")
debug("stopping node 2")
node2.stop(gently=False)
debug("inserting data in nodes 1 and 3")
insert_c1c2(session,
keys=range(100, 150),
consistency=ConsistencyLevel.TWO)
node1.flush()
node3.flush()
debug("start and repair node 2")
node2.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node2.repair()
else:
node2.nodetool("repair -par -inc")
debug("replace node and check data integrity")
node3.stop(gently=False)
node5 = Node('node5', cluster, True, ('127.0.0.5', 9160),
('127.0.0.5', 7000), '7500', '0', None,
('127.0.0.5', 9042))
cluster.add(node5, False)
node5.start(replace_address='127.0.0.3', wait_other_notice=True)
assert_one(session, "SELECT COUNT(*) FROM ks.cf LIMIT 200", [149])
class TestTransientReplicationRing(Tester):
keyspace = "ks"
table = "tbl"
def select(self):
return "SELECT * from %s.%s" % (self.keyspace, self.table)
def select_statement(self):
return SimpleStatement(self.select(), consistency_level=NODELOCAL)
def point_select(self):
return "SELECT * from %s.%s where pk = %%s" % (self.keyspace,
self.table)
def point_select_statement(self):
return SimpleStatement(self.point_select(),
consistency_level=NODELOCAL)
def check_expected(self, sessions, expected, node=None, cleanup=False):
"""Check that each node has the expected values present"""
if node is None:
node = list(range(1000))
for idx, session, expect, node in zip(range(0, 1000), sessions,
expected, node):
print("Checking idx " + str(idx))
print(
str([row for row in session.execute(self.select_statement())]))
if cleanup:
node.nodetool('cleanup')
assert_all(session, self.select(), expect, cl=NODELOCAL)
def check_replication(self, sessions, exactly=None, gte=None, lte=None):
"""Assert that the test values are replicated a required number of times"""
for i in range(0, 40):
count = 0
for session in sessions:
for _ in session.execute(self.point_select_statement(),
["%05d" % i]):
count += 1
if exactly:
assert count == exactly, "Wrong replication for %05d should be exactly %d" % (
i, exactly)
if gte:
assert count >= gte, "Count for %05d should be >= %d" % (i,
gte)
if lte:
assert count <= lte, "Count for %05d should be <= %d" % (i,
lte)
@pytest.fixture
def cheap_quorums(self):
return False
@pytest.fixture(scope='function', autouse=True)
def setup_cluster(self, fixture_dtest_setup):
self.tokens = ['00010', '00020', '00030']
patch_start(self.cluster)
self.cluster.set_configuration_options(
values={
'hinted_handoff_enabled':
False,
'num_tokens':
1,
'commitlog_sync_period_in_ms':
500,
'enable_transient_replication':
True,
'partitioner':
'org.apache.cassandra.dht.OrderPreservingPartitioner'
})
print("CLUSTER INSTALL DIR: ")
print(self.cluster.get_install_dir())
self.cluster.populate(3,
tokens=self.tokens,
debug=True,
install_byteman=True)
# self.cluster.populate(3, debug=True, install_byteman=True)
self.cluster.start(
jvm_args=['-Dcassandra.enable_nodelocal_queries=true'])
# enable shared memory
for node in self.cluster.nodelist():
patch_start(node)
print(node.logfilename())
self.nodes = self.cluster.nodelist()
self.node1, self.node2, self.node3 = self.nodes
session = self.exclusive_cql_connection(self.node3)
replication_params = OrderedDict()
replication_params['class'] = 'NetworkTopologyStrategy'
replication_params['datacenter1'] = '3/1'
replication_params = ', '.join("'%s': '%s'" % (k, v)
for k, v in replication_params.items())
session.execute("CREATE KEYSPACE %s WITH REPLICATION={%s}" %
(self.keyspace, replication_params))
print("CREATE KEYSPACE %s WITH REPLICATION={%s}" %
(self.keyspace, replication_params))
session.execute(
"CREATE TABLE %s.%s (pk varchar, ck int, value int, PRIMARY KEY (pk, ck)) WITH speculative_retry = 'NEVER' AND additional_write_policy = 'NEVER' AND read_repair = 'NONE'"
% (self.keyspace, self.table))
def quorum(self, session, stmt_str):
return session.execute(
SimpleStatement(stmt_str,
consistency_level=ConsistencyLevel.QUORUM))
def insert_row(self, pk, ck, value, session=None, node=None):
session = session or self.exclusive_cql_connection(node or self.node1)
# token = BytesToken.from_key(pack('>i', pk)).value
# assert token < BytesToken.from_string(self.tokens[0]).value or BytesToken.from_string(self.tokens[-1]).value < token # primary replica should be node1
# TODO Is quorum really right? I mean maybe we want ALL with retries since we really don't want to the data
# not at a replica unless it is intentional
self.quorum(
session,
"INSERT INTO %s.%s (pk, ck, value) VALUES ('%05d', %s, %s)" %
(self.keyspace, self.table, pk, ck, value))
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_bootstrap_and_cleanup(self):
"""Test bootstrapping a new node across a mix of repaired and unrepaired data"""
main_session = self.patient_cql_connection(self.node1)
nodes = [self.node1, self.node2, self.node3]
for i in range(0, 40, 2):
self.insert_row(i, i, i, main_session)
sessions = [
self.exclusive_cql_connection(node)
for node in [self.node1, self.node2, self.node3]
]
expected = [
gen_expected(range(0, 11, 2), range(22, 40, 2)),
gen_expected(range(0, 22, 2), range(32, 40, 2)),
gen_expected(range(12, 31, 2))
]
self.check_expected(sessions, expected)
# Make sure at least a little data is repaired, this shouldn't move data anywhere
repair_nodes(nodes)
self.check_expected(sessions, expected)
# Ensure that there is at least some transient data around, because of this if it's missing after bootstrap
# We know we failed to get it from the transient replica losing the range entirely
nodes[1].stop(wait_other_notice=True)
for i in range(1, 40, 2):
self.insert_row(i, i, i, main_session)
nodes[1].start(wait_for_binary_proto=True)
sessions = [
self.exclusive_cql_connection(node)
for node in [self.node1, self.node2, self.node3]
]
expected = [
gen_expected(range(0, 11), range(11, 20, 2), range(21, 40)),
gen_expected(range(0, 21, 2), range(32, 40, 2)),
gen_expected(range(1, 11, 2), range(11, 31), range(31, 40, 2))
]
# Every node should have some of its fully replicated data and one and two should have some transient data
self.check_expected(sessions, expected)
node4 = new_node(self.cluster, bootstrap=True, token='00040')
patch_start(node4)
nodes.append(node4)
node4.start(wait_for_binary_proto=True)
expected.append(gen_expected(range(11, 20, 2), range(21, 40)))
sessions.append(self.exclusive_cql_connection(node4))
# Because repair was never run and nodes had transient data it will have data for transient ranges (node1, 11-20)
assert_all(sessions[3], self.select(), expected[3], cl=NODELOCAL)
# Node1 no longer transiently replicates 11-20, so cleanup will clean it up
# Node1 also now transiently replicates 21-30 and half the values in that range were repaired
expected[0] = gen_expected(range(0, 11), range(21, 30, 2),
range(31, 40))
# Node2 still missing data since it was down during some insertions, it also lost some range (31-40)
expected[1] = gen_expected(range(0, 21, 2))
expected[2] = gen_expected(range(1, 11, 2), range(11, 31))
# Cleanup should only impact if a node lost a range entirely or started to transiently replicate it and the data
# was repaired
self.check_expected(sessions, expected, nodes, cleanup=True)
repair_nodes(nodes)
expected = [
gen_expected(range(0, 11), range(31, 40)),
gen_expected(range(0, 21)),
gen_expected(range(11, 31)),
gen_expected(range(21, 40))
]
self.check_expected(sessions, expected, nodes, cleanup=True)
# Every value should be replicated exactly 2 times
self.check_replication(sessions, exactly=2)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def move_test(self, move_token, expected_after_move,
expected_after_repair):
"""Helper method to run a move test cycle"""
node4 = new_node(self.cluster, bootstrap=True, token='00040')
patch_start(node4)
node4.start(wait_for_binary_proto=True)
main_session = self.patient_cql_connection(self.node1)
nodes = [self.node1, self.node2, self.node3, node4]
for i in range(0, 40, 2):
print("Inserting " + str(i))
self.insert_row(i, i, i, main_session)
# Make sure at least a little data is repaired
repair_nodes(nodes)
# Ensure that there is at least some transient data around, because of this if it's missing after bootstrap
# We know we failed to get it from the transient replica losing the range entirely
nodes[1].stop(wait_other_notice=True)
for i in range(1, 40, 2):
print("Inserting " + str(i))
self.insert_row(i, i, i, main_session)
nodes[1].start(wait_for_binary_proto=True)
sessions = [
self.exclusive_cql_connection(node)
for node in [self.node1, self.node2, self.node3, node4]
]
expected = [
gen_expected(range(0, 11), range(31, 40)),
gen_expected(range(0, 21, 2)),
gen_expected(range(1, 11, 2), range(11, 31)),
gen_expected(range(11, 20, 2), range(21, 40))
]
self.check_expected(sessions, expected)
self.check_replication(sessions, exactly=2)
nodes[0].nodetool('move %s' % move_token)
cleanup_nodes(nodes)
self.check_replication(sessions, gte=2, lte=3)
self.check_expected(sessions, expected=expected_after_move)
repair_nodes(nodes)
self.check_expected(sessions,
expected_after_repair,
nodes,
cleanup=True)
self.check_replication(sessions, exactly=2)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_move_forwards_between_and_cleanup(self):
"""Test moving a node forwards past a neighbor token"""
move_token = '00025'
expected_after_move = [
gen_expected(range(0, 26), range(31, 40, 2)),
gen_expected(range(0, 21, 2), range(31, 40)),
gen_expected(range(1, 11, 2), range(11, 21, 2), range(21, 31)),
gen_expected(range(21, 26, 2), range(26, 40))
]
expected_after_repair = [
gen_expected(range(0, 26)),
gen_expected(range(0, 21), range(31, 40)),
gen_expected(range(21, 31), ),
gen_expected(range(26, 40))
]
self.move_test(move_token, expected_after_move, expected_after_repair)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_move_forwards_and_cleanup(self):
"""Test moving a node forwards without going past a neighbor token"""
move_token = '00015'
expected_after_move = [
gen_expected(range(0, 16), range(31, 40)),
gen_expected(range(0, 21, 2)),
gen_expected(range(1, 16, 2), range(16, 31)),
gen_expected(range(17, 20, 2), range(21, 40))
]
expected_after_repair = [
gen_expected(range(0, 16), range(31, 40)),
gen_expected(range(0, 21)),
gen_expected(range(16, 31)),
gen_expected(range(21, 40))
]
self.move_test(move_token, expected_after_move, expected_after_repair)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_move_backwards_between_and_cleanup(self):
"""Test moving a node backwards past it's preceding neighbor's token"""
move_token = '00035'
expected_after_move = [
gen_expected(range(1, 21, 2), range(21, 36)),
gen_expected(range(0, 21, 2), range(36, 40)),
gen_expected(range(0, 31), range(37, 40, 2)),
gen_expected(range(21, 30, 2), range(31, 40))
]
expected_after_repair = [
gen_expected(range(21, 36)),
gen_expected(range(0, 21), range(36, 40)),
gen_expected(range(0, 31)),
gen_expected(range(31, 40))
]
self.move_test(move_token, expected_after_move, expected_after_repair)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_move_backwards_and_cleanup(self):
"""Test moving a node backwards without moving past a neighbor token"""
move_token = '00005'
expected_after_move = [
gen_expected(range(0, 6), range(31, 40)),
gen_expected(range(0, 21, 2)),
gen_expected(range(1, 6, 2), range(6, 31)),
gen_expected(range(7, 20, 2), range(21, 40))
]
expected_after_repair = [
gen_expected(range(0, 6), range(31, 40)),
gen_expected(range(0, 21)),
gen_expected(range(6, 31)),
gen_expected(range(21, 40))
]
self.move_test(move_token, expected_after_move, expected_after_repair)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_decommission(self):
"""Test decommissioning a node correctly streams out all the data"""
node4 = new_node(self.cluster, bootstrap=True, token='00040')
patch_start(node4)
node4.start(wait_for_binary_proto=True)
main_session = self.patient_cql_connection(self.node1)
nodes = [self.node1, self.node2, self.node3, node4]
for i in range(0, 40, 2):
print("Inserting " + str(i))
self.insert_row(i, i, i, main_session)
# Make sure at least a little data is repaired
repair_nodes(nodes)
# Ensure that there is at least some transient data around, because of this if it's missing after bootstrap
# We know we failed to get it from the transient replica losing the range entirely
nodes[1].stop(wait_other_notice=True)
for i in range(1, 40, 2):
print("Inserting " + str(i))
self.insert_row(i, i, i, main_session)
nodes[1].start(wait_for_binary_proto=True)
sessions = [
self.exclusive_cql_connection(node)
for node in [self.node1, self.node2, self.node3, node4]
]
expected = [
gen_expected(range(0, 11), range(31, 40)),
gen_expected(range(0, 21, 2)),
gen_expected(range(1, 11, 2), range(11, 31)),
gen_expected(range(11, 20, 2), range(21, 40))
]
self.check_expected(sessions, expected)
# node1 has transient data we want to see streamed out on move
nodes[3].nodetool('decommission')
nodes = nodes[:-1]
sessions = sessions[:-1]
expected = [
gen_expected(range(0, 11), range(11, 21, 2), range(21, 40)),
gen_expected(range(0, 21, 2), range(21, 30, 2), range(31, 40)),
gen_expected(range(1, 11, 2), range(11, 31), range(31, 40, 2))
]
cleanup_nodes(nodes)
self.check_replication(sessions, gte=2, lte=3)
self.check_expected(sessions, expected)
repair_nodes(nodes)
# There should be no transient data anywhere
expected = [
gen_expected(range(0, 11), range(21, 40)),
gen_expected(range(0, 21), range(31, 40)),
gen_expected(range(11, 31))
]
self.check_expected(sessions, expected, nodes, cleanup=True)
self.check_replication(sessions, exactly=2)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_remove(self):
"""Test a mix of ring change operations across a mix of transient and repaired/unrepaired data"""
node4 = new_node(self.cluster, bootstrap=True, token='00040')
patch_start(node4)
node4.start(wait_for_binary_proto=True)
main_session = self.patient_cql_connection(self.node1)
nodes = [self.node1, self.node2, self.node3]
# We want the node being removed to have no data on it
# so nodetool remove always gets all the necessary data from survivors
node4_id = node4.nodetool('info').stdout[25:61]
node4.stop(wait_other_notice=True)
for i in range(0, 40):
self.insert_row(i, i, i, main_session)
sessions = [
self.exclusive_cql_connection(node)
for node in [self.node1, self.node2, self.node3]
]
expected = [
gen_expected(range(0, 11), range(21, 40)),
gen_expected(range(0, 21), range(31, 40)),
gen_expected(range(11, 31))
]
# Every node should some of its fully replicated data and one and two should have some transient data
self.check_expected(sessions, expected)
nodes[0].nodetool('removenode ' + node4_id)
# Give streaming time to occur, it's asynchronous from removenode completing at other nodes
import time
time.sleep(15)
self._everyone_should_have_everything(sessions)
repair_nodes(nodes)
cleanup_nodes(nodes)
self._nodes_have_proper_ranges_after_repair_and_cleanup(sessions)
@flaky(max_runs=1)
@pytest.mark.no_vnodes
def test_replace(self):
main_session = self.patient_cql_connection(self.node1)
# We want the node being replaced to have no data on it so the replacement definitely fetches all the data
self.node2.stop(wait_other_notice=True)
for i in range(0, 40):
self.insert_row(i, i, i, main_session)
replacement_address = self.node2.address()
self.node2.stop(wait_other_notice=True)
self.cluster.remove(self.node2)
self.node2 = Node('replacement',
cluster=self.cluster,
auto_bootstrap=True,
thrift_interface=None,
storage_interface=(replacement_address, 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=(replacement_address, 9042))
patch_start(self.node2)
nodes = [self.node1, self.node2, self.node3]
self.cluster.add(self.node2, False, data_center='datacenter1')
jvm_args = [
"-Dcassandra.replace_address=%s" % replacement_address,
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"
]
self.node2.start(jvm_args=jvm_args, wait_for_binary_proto=True)
sessions = [
self.exclusive_cql_connection(node)
for node in [self.node1, self.node2, self.node3]
]
self._everyone_should_have_everything(sessions)
repair_nodes(nodes)
cleanup_nodes(nodes)
self._nodes_have_proper_ranges_after_repair_and_cleanup(sessions)
def _everyone_should_have_everything(self, sessions):
expected = [gen_expected(range(0, 40))] * 3
self.check_replication(sessions, exactly=3)
self.check_expected(sessions, expected)
def _nodes_have_proper_ranges_after_repair_and_cleanup(self, sessions):
expected = [
gen_expected(range(0, 11), range(21, 40)),
gen_expected(range(0, 21), range(31, 40)),
gen_expected(range(11, 31))
]
self.check_replication(sessions, exactly=2)
self.check_expected(sessions, expected)
def repair_compaction_fine_test(self):
"""Check that we do not stream data for repairs past the last repair.
Check cases i) node goes down, data inserted, node comes up run repair
ii) after case i, do similar with a different node down (since compaction has seperated repaired)
iii) check that repair works appropriately where a new node is replacing
"""
cluster = self.cluster
cluster.set_configuration_options(values={
'hinted_handoff_enabled': False,
'auto_snapshot': False
},
batch_commitlog=False)
cluster.populate(3).start()
[node1, node2, node3] = cluster.nodelist()
cursor = self.patient_cql_connection(node1)
self.create_ks(cursor, 'ks', 3)
self.create_cf(cursor,
'cf',
read_repair=0.0,
columns={
'c1': 'text',
'c2': 'text'
})
debug("insert data into all")
for x in range(1, 5):
insert_c1c2(cursor, x, ConsistencyLevel.ALL)
node1.flush()
debug("bringing down node 3")
node3.flush()
node3.stop(gently=False)
debug("inserting additional data into node 1 and 2")
for y in range(5, 10):
insert_c1c2(cursor, y, ConsistencyLevel.TWO)
node1.flush()
node2.flush()
debug("restarting and repairing node 3")
node3.start()
node3.repair()
sstableNode1 = node3.grep_log(
"reading file from /127.0.0.1, repairedAt = 0")
sstableNode2 = node3.grep_log(
"reading file from /127.0.0.2, repairedAt = 0")
catchBadSSReads = node3.grep_log(
"reading file from .* repairedAt = ([1-9])")
self.assertGreaterEqual(len(sstableNode1), 1)
self.assertGreaterEqual(len(sstableNode2), 1)
self.assertLess(len(catchBadSSReads), 1)
debug("stopping node 2")
node2.stop(gently=False)
debug("inserting data in nodes 1 and 3")
for z in range(10, 15):
insert_c1c2(cursor, z, ConsistencyLevel.TWO)
node1.flush()
debug("start and repair node 2")
node2.flush()
node2.start()
node2.repair()
fileFromNode1 = node2.grep_log(
"reading file from /127.0.0.1, repairedAt = 0")
fileFromNode3 = node2.grep_log(
"reading file from /127.0.0.3, repairedAt = 0")
catchBadReads = node2.grep_log(
"reading file from .* repairedAt = ([1-9])")
self.assertGreaterEqual(len(fileFromNode1), 1)
self.assertGreaterEqual(len(fileFromNode3), 1)
self.assertLess(len(catchBadReads), 1)
node4 = Node('node4', cluster, True, ('127.0.0.4', 9160),
('127.0.0.4', 7000), '7400', '0', None,
('127.0.0.4', 9042))
node4.start()
debug("replace node and check repair-like process")
node3.stop(gently=False)
node5 = Node('node5', cluster, True, ('127.0.0.5', 9160),
('127.0.0.5', 7000), '7500', '0', None,
('127.0.0.5', 9042))
cluster.add(node5, False)
node5.start(replace_address='127.0.0.3', wait_other_notice=True)
fileRead = node5.grep_log("reading file from .*, repairedAt = 0")
self.assertGreaterEqual(len(fileRead), 1)
# additionally should see 14 distinct keys in data(this prints to command line)
debug((node2.run_sstable2json()))
rows = cursor.execute("SELECT COUNT(*) FROM ks.cf LIMIT 100")
results = rows[0]
debug(results)
self.assertEqual(results[0], 14)
def _replace_node_test(self, gently):
"""
Check that the replace address function correctly replaces a node that has failed in a cluster.
Create a cluster, cause a node to fail, and bring up a new node with the replace_address parameter.
Check that tokens are migrated and that data is replicated properly.
"""
debug("Starting cluster with 3 nodes.")
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
if DISABLE_VNODES:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(node3.get_conf_option('num_tokens'))
debug("testing with num_tokens: {}".format(num_tokens))
debug("Inserting Data...")
node1.stress(['write', 'n=10K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
session.default_timeout = 45
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
# stop node, query should not work with consistency 3
debug("Stopping node 3.")
node3.stop(gently=gently, wait_other_notice=True)
debug("Testing node stoppage (query should fail).")
with self.assertRaises(NodeUnavailable):
try:
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
session.execute(query)
except (Unavailable, ReadTimeout):
raise NodeUnavailable("Node could not be queried.")
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
cluster.add(node4, False)
node4.start(replace_address='127.0.0.3', wait_for_binary_proto=True)
debug("Verifying tokens migrated sucessfully")
moved_tokens = node4.grep_log("Token .* changing ownership from /127.0.0.3 to /127.0.0.4")
debug("number of moved tokens: {}".format(len(moved_tokens)))
self.assertEqual(len(moved_tokens), num_tokens)
# check that restarting node 3 doesn't work
debug("Try to restart node 3 (should fail)")
node3.start(wait_other_notice=False)
collision_log = node1.grep_log("between /127.0.0.3 and /127.0.0.4; /127.0.0.4 is the new owner")
debug(collision_log)
self.assertEqual(len(collision_log), 1)
node3.stop(gently=False)
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def test_multiple_repair(self):
"""
* Launch a three node cluster
* Create a keyspace with RF 3 and a table
* Insert 49 rows
* Stop node3
* Insert 50 more rows
* Restart node3
* Issue an incremental repair on node3
* Stop node2
* Insert a final50 rows
* Restart node2
* Issue an incremental repair on node2
* Replace node3 with a new node
* Verify data integrity
# TODO: Several more verifications of data need to be interspersed throughout the test. The final assertion is insufficient.
@jira_ticket CASSANDRA-10644
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node1)
create_ks(session, 'ks', 3)
if cluster.version() < '4.0':
create_cf(session,
'cf',
read_repair=0.0,
columns={
'c1': 'text',
'c2': 'text'
})
else:
create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
logger.debug("insert data")
insert_c1c2(session,
keys=list(range(1, 50)),
consistency=ConsistencyLevel.ALL)
node1.flush()
logger.debug("bringing down node 3")
node3.flush()
node3.stop(gently=False)
logger.debug("inserting additional data into node 1 and 2")
insert_c1c2(session,
keys=list(range(50, 100)),
consistency=ConsistencyLevel.TWO)
node1.flush()
node2.flush()
logger.debug("restarting and repairing node 3")
node3.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node3.repair()
else:
node3.nodetool("repair -par -inc")
# wait stream handlers to be closed on windows
# after session is finished (See CASSANDRA-10644)
if is_win:
time.sleep(2)
logger.debug("stopping node 2")
node2.stop(gently=False)
logger.debug("inserting data in nodes 1 and 3")
insert_c1c2(session,
keys=list(range(100, 150)),
consistency=ConsistencyLevel.TWO)
node1.flush()
node3.flush()
logger.debug("start and repair node 2")
node2.start(wait_for_binary_proto=True)
if cluster.version() >= "2.2":
node2.repair()
else:
node2.nodetool("repair -par -inc")
logger.debug("replace node and check data integrity")
node3.stop(gently=False)
node5 = Node('node5', cluster, True, ('127.0.0.5', 9160),
('127.0.0.5', 7000), '7500', '0', None,
('127.0.0.5', 9042))
cluster.add(node5, False, data_center="dc1")
node5.start(replace_address='127.0.0.3')
assert_one(session, "SELECT COUNT(*) FROM ks.cf LIMIT 200", [149])
def test_node_cannot_join_as_hibernating_node_without_replace_address(self):
"""
@jira_ticket CASSANDRA-14559
Test that a node cannot bootstrap without replace_address if a hibernating node exists with that address
"""
cluster = self.cluster
cluster.set_environment_variable('CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2)
# Setting seed node to first node to make sure replaced node is not in own seed list
cluster.set_configuration_options({
'seed_provider': [{'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider',
'parameters': [{'seeds': '127.0.0.1'}]
}]
})
cluster.start()
node1 = cluster.nodelist()[0]
node2 = cluster.nodelist()[1]
replacement_address = node2.address()
node2.stop()
jvm_option = 'replace_address'
logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option,
replacement_address))
replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None,
binary_interface=(replacement_address, 9042))
cluster.add(replacement_node, False)
extra_jvm_args = []
extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replacement_address),
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"])
wait_other_notice = False
wait_for_binary_proto = False
# Killing node earlier in bootstrap to prevent node making it to 'normal' status.
t = KillOnReadyToBootstrap(replacement_node)
t.start()
replacement_node.start(jvm_args=extra_jvm_args,
wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice)
t.join()
logger.debug("Asserting that original replacement node is not running")
assert not replacement_node.is_running()
# Assert node is actually in hibernate for test to be accurate.
logger.debug("Asserting that node is actually in hibernate status for test accuracy")
assert 'hibernate' in node1.nodetool("gossipinfo").stdout
extra_jvm_args = []
extra_jvm_args.extend(["-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"])
logger.debug("Starting blind replacement node {}".format(replacement_address))
blind_replacement_node = Node('blind_replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None,
binary_interface=(replacement_address, 9042))
cluster.add(blind_replacement_node, False)
wait_other_notice = False
wait_for_binary_proto = False
blind_replacement_node.start(wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice)
# Asserting that the new node has correct log entry
self.assert_log_had_msg(blind_replacement_node, "A node with the same IP in hibernate status was detected", timeout=60)
# Waiting two seconds to give node a chance to stop in case above assertion is True.
# When this happens cassandra may not shut down fast enough and the below assertion fails.
time.sleep(15)
# Asserting that then new node is not running.
# This tests the actual expected state as opposed to just checking for the existance of the above error message.
assert not blind_replacement_node.is_running()
class BaseReplaceAddressTest(Tester):
@pytest.fixture(autouse=True)
def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
fixture_dtest_setup.ignore_log_patterns = (
# This one occurs when trying to send the migration to a
# node that hasn't started yet, and when it does, it gets
# replayed and everything is fine.
r'Can\'t send migration request: node.*is down',
r'Migration task failed to complete', # 10978
# ignore streaming error during bootstrap
r'Streaming error occurred',
r'failed stream session',
r'Failed to properly handshake with peer'
)
def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False):
logger.debug("Starting cluster with {} nodes.".format(n))
self.cluster.populate(n)
if opts is not None:
logger.debug("Setting cluster options: {}".format(opts))
self.cluster.set_configuration_options(opts)
self.cluster.set_batch_commitlog(enabled=True)
self.query_node = self.cluster.nodelist()[0]
self.replaced_node = self.cluster.nodelist()[-1]
self.cluster.seeds.remove(self.replaced_node)
NUM_TOKENS = os.environ.get('NUM_TOKENS', '256')
if not self.dtest_config.use_vnodes:
self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': 1})
else:
self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': NUM_TOKENS})
if enable_byteman:
# set up byteman
self.query_node.byteman_port = '8100'
self.query_node.import_config_files()
if mixed_versions:
logger.debug("Starting nodes on version 2.2.4")
self.cluster.set_install_dir(version="2.2.4")
self.cluster.start()
if self.cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(self.query_node)
# Change system_auth keyspace replication factor to 2, otherwise replace will fail
session.execute("""ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy',
'replication_factor':2};""")
def _do_replace(self, same_address=False, jvm_option='replace_address',
wait_other_notice=False, wait_for_binary_proto=True,
replace_address=None, opts=None, data_center=None,
extra_jvm_args=None):
if replace_address is None:
replace_address = self.replaced_node.address()
# only create node if it's not yet created
if self.replacement_node is None:
replacement_address = '127.0.0.4'
if same_address:
replacement_address = self.replaced_node.address()
self.cluster.remove(self.replaced_node)
logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address))
self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True,
thrift_interface=None, storage_interface=(replacement_address, 7000),
jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042))
if opts is not None:
logger.debug("Setting options on replacement node: {}".format(opts))
self.replacement_node.set_configuration_options(opts)
self.cluster.add(self.replacement_node, False, data_center=data_center)
if extra_jvm_args is None:
extra_jvm_args = []
extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address),
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"])
self.replacement_node.start(jvm_args=extra_jvm_args,
wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice)
if self.cluster.cassandra_version() >= '2.2.8' and same_address:
self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement",
timeout=60)
def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE):
if self.replaced_node.is_running():
logger.debug("Stopping {}".format(self.replaced_node.name))
self.replaced_node.stop(gently=gently, wait_other_notice=True)
logger.debug("Testing node stoppage (query should fail).")
with pytest.raises((Unavailable, ReadTimeout)):
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {}'.format(table), consistency_level=cl)
session.execute(query)
def _insert_data(self, n='1k', rf=3, whitelist=False):
logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf))
self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf),
'-rate', 'threads=10'],
whitelist=whitelist)
self.cluster.flush()
time.sleep(20)
def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000):
logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit))
session = self.patient_cql_connection(self.query_node)
query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl)
return rows_to_list(session.execute(query, timeout=20))
def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000,
restart_nodes=False):
assert len(initial_data) > 0, "Initial data must be greater than 0"
# query should work again
logger.debug("Stopping old nodes")
for node in self.cluster.nodelist():
if node.is_running() and node != self.replacement_node:
logger.debug("Stopping {}".format(node.name))
node.stop(gently=False, wait_other_notice=True)
logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit))
session = self.patient_exclusive_cql_connection(self.replacement_node)
assert_all(session, 'select * from {} LIMIT {}'.format(table, limit),
expected=initial_data,
cl=cl)
def _verify_replacement(self, node, same_address):
if not same_address:
if self.cluster.cassandra_version() >= '2.2.7':
address_prefix = '' if self.cluster.cassandra_version() >= '4.0' else '/'
node.watch_log_for("Node {}{} is replacing {}{}"
.format(address_prefix, self.replacement_node.address_for_current_version(),
address_prefix, self.replaced_node.address_for_current_version()),
timeout=60, filename='debug.log')
node.watch_log_for("Node {}{} will complete replacement of {}{} for tokens"
.format(address_prefix, self.replacement_node.address_for_current_version(),
address_prefix, self.replaced_node.address_for_current_version()), timeout=10)
node.watch_log_for("removing endpoint {}{}".format(address_prefix, self.replaced_node.address_for_current_version()),
timeout=60, filename='debug.log')
else:
node.watch_log_for("between /{} and /{}; /{} is the new owner"
.format(self.replaced_node.address(),
self.replacement_node.address(),
self.replacement_node.address()),
timeout=60)
def _verify_tokens_migrated_successfully(self, previous_log_size=None):
if not self.dtest_config.use_vnodes:
num_tokens = 1
else:
# a little hacky but grep_log returns the whole line...
num_tokens = int(self.replacement_node.get_conf_option('num_tokens'))
logger.debug("Verifying {} tokens migrated sucessfully".format(num_tokens))
logs = self.replacement_node.grep_log(r"Token (.*?) changing ownership from /{} to /{}"
.format(self.replaced_node.address(),
self.replacement_node.address()))
if (previous_log_size is not None):
assert len(logs) == previous_log_size
moved_tokens = set([l[1].group(1) for l in logs])
logger.debug("number of moved tokens: {}".format(len(moved_tokens)))
assert len(moved_tokens) == num_tokens
return len(logs)
def _test_insert_data_during_replace(self, same_address, mixed_versions=False):
"""
@jira_ticket CASSANDRA-8523
"""
default_install_dir = self.cluster.get_install_dir()
self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions)
self._insert_data(n='1k')
initial_data = self._fetch_initial_data()
self._stop_node_to_replace()
if mixed_versions:
logger.debug("Upgrading all except {} to current version".format(self.query_node.address()))
self.cluster.set_install_dir(install_dir=default_install_dir)
for node in self.cluster.nodelist():
if node.is_running() and node != self.query_node:
logger.debug("Upgrading {} to current version".format(node.address()))
node.stop(gently=True, wait_other_notice=True)
node.start(wait_other_notice=True, wait_for_binary_proto=True)
# start node in current version on write survey mode
self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"])
# Insert additional keys on query node
self._insert_data(n='2k', whitelist=True)
# If not same address or mixed versions, query node should forward writes to replacement node
# so we update initial data to reflect additional keys inserted during replace
if not same_address and not mixed_versions:
initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO)
logger.debug("Joining replaced node")
self.replacement_node.nodetool("join")
if not same_address:
for node in self.cluster.nodelist():
# if mixed version, query node is not upgraded so it will not print replacement log
if node.is_running() and (not mixed_versions or node != self.query_node):
self._verify_replacement(node, same_address)
self._verify_data(initial_data)
