def replace_with_reset_resume_state_test(self): """Test replace with resetting bootstrap progress""" cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)']) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE) initial_data = rows_to_list(session.execute(query)) node3.stop(gently=False) # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # replace node 3 with node 4 debug("Starting node 4 to replace node 3") node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160), storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=('127.0.0.4', 9042)) # keep timeout low so that test won't hang node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000}) cluster.add(node4, False) try: node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False) except NodeError: pass # node doesn't start as expected t.join() node1.start() # restart node4 bootstrap with resetting bootstrap state node4.stop() mark = node4.mark_log() node4.start(jvm_args=[ "-Dcassandra.replace_address_first_boot=127.0.0.3", "-Dcassandra.reset_bootstrap_progress=true" ]) # check if we reset bootstrap state node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark) # wait for node3 ready to query node4.watch_log_for("Listening for thrift clients...", from_mark=mark) # check if 2nd bootstrap succeeded assert_bootstrap_state(self, node4, 'COMPLETED') # query should work again debug("Stopping old nodes") node1.stop(gently=False, wait_other_notice=True) node2.stop(gently=False, wait_other_notice=True) debug("Verifying data on new node.") session = self.patient_exclusive_cql_connection(node4) assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table), expected=initial_data, cl=ConsistencyLevel.ONE)
def resumable_replace_test(self): """ Test resumable bootstrap while replacing node. Feature introduced in 2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838 @jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838 """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)']) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE) initial_data = rows_to_list(session.execute(query)) node3.stop(gently=False) # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # replace node 3 with node 4 debug("Starting node 4 to replace node 3") node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160), storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=('127.0.0.4', 9042)) # keep timeout low so that test won't hang node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000}) cluster.add(node4, False) try: node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False) except NodeError: pass # node doesn't start as expected t.join() # bring back node1 and invoke nodetool bootstrap to resume bootstrapping node1.start() node4.nodetool('bootstrap resume') # check if we skipped already retrieved ranges node4.watch_log_for("already available. Skipping streaming.") # wait for node3 ready to query node4.watch_log_for("Listening for thrift clients...") # check if 2nd bootstrap succeeded assert_bootstrap_state(self, node4, 'COMPLETED') # query should work again debug("Stopping old nodes") node1.stop(gently=False, wait_other_notice=True) node2.stop(gently=False, wait_other_notice=True) debug("Verifying data on new node.") session = self.patient_exclusive_cql_connection(node4) assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table), expected=initial_data, cl=ConsistencyLevel.ONE)
def replace_with_reset_resume_state_test(self): """Test replace with resetting bootstrap progress""" cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() node1.stress(['write', 'n=100000', '-schema', 'replication(factor=3)']) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE) initialData = list(session.execute(query)) node3.stop(gently=False) # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # replace node 3 with node 4 debug("Starting node 4 to replace node 3") node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042)) # keep timeout low so that test won't hang node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000}) cluster.add(node4, False) try: node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False) except NodeError: pass # node doesn't start as expected t.join() node1.start() # restart node4 bootstrap with resetting bootstrap state node4.stop() mark = node4.mark_log() node4.start(jvm_args=[ "-Dcassandra.replace_address_first_boot=127.0.0.3", "-Dcassandra.reset_bootstrap_progress=true" ]) # check if we reset bootstrap state node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark) # wait for node3 ready to query node4.watch_log_for("Listening for thrift clients...", from_mark=mark) # check if 2nd bootstrap succeeded session = self.exclusive_cql_connection(node4) rows = list(session.execute("SELECT bootstrapped FROM system.local WHERE key='local'")) assert len(rows) == 1 assert rows[0][0] == 'COMPLETED', rows[0][0] # query should work again debug("Verifying querying works again.") finalData = list(session.execute(query)) self.assertListEqual(initialData, finalData)
def resumable_replace_test(self): """ Test resumable bootstrap while replacing node. Feature introduced in 2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838 @jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838 """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() node1.stress(['write', 'n=100K', '-schema', 'replication(factor=3)']) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE) initialData = list(session.execute(query)) node3.stop(gently=False) # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # replace node 3 with node 4 debug("Starting node 4 to replace node 3") node4 = Node('node4', cluster, True, ('127.0.0.4', 9160), ('127.0.0.4', 7000), '7400', '0', None, binary_interface=('127.0.0.4', 9042)) # keep timeout low so that test won't hang node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000}) cluster.add(node4, False) try: node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False) except NodeError: pass # node doesn't start as expected t.join() # bring back node1 and invoke nodetool bootstrap to resume bootstrapping node1.start() node4.nodetool('bootstrap resume') # check if we skipped already retrieved ranges node4.watch_log_for("already available. Skipping streaming.") # wait for node3 ready to query node4.watch_log_for("Listening for thrift clients...") # check if 2nd bootstrap succeeded session = self.exclusive_cql_connection(node4) rows = list(session.execute("SELECT bootstrapped FROM system.local WHERE key='local'")) assert len(rows) == 1 assert rows[0][0] == 'COMPLETED', rows[0][0] # query should work again debug("Verifying querying works again.") finalData = list(session.execute(query)) self.assertListEqual(initialData, finalData)
def configure_node(self, node: Node, index): # set dc/rack manually, since CCM doesn't support custom racks node.set_configuration_options({ 'endpoint_snitch': 'GossipingPropertyFileSnitch', }) rackdc_path = Path( node.get_conf_dir()) / 'cassandra-rackdc.properties' with open(rackdc_path, 'w') as f: f.write(f'dc={node.dc}\nrack={node.rack}\n')
class BaseReplaceAddressTest(Tester): @pytest.fixture(autouse=True) def fixture_add_additional_log_patterns(self, fixture_dtest_setup): fixture_dtest_setup.ignore_log_patterns = ( # This one occurs when trying to send the migration to a # node that hasn't started yet, and when it does, it gets # replayed and everything is fine. r'Can\'t send migration request: node.*is down', r'Migration task failed to complete', # 10978 # ignore streaming error during bootstrap r'Streaming error occurred', r'failed stream session', r'Failed to properly handshake with peer' ) def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False): logger.debug("Starting cluster with {} nodes.".format(n)) self.cluster.populate(n) if opts is not None: logger.debug("Setting cluster options: {}".format(opts)) self.cluster.set_configuration_options(opts) self.cluster.set_batch_commitlog(enabled=True) self.query_node = self.cluster.nodelist()[0] self.replaced_node = self.cluster.nodelist()[-1] self.cluster.seeds.remove(self.replaced_node) if enable_byteman: # set up byteman self.query_node.byteman_port = '8100' self.query_node.import_config_files() if mixed_versions: logger.debug("Starting nodes on version 2.2.4") self.cluster.set_install_dir(version="2.2.4") self.cluster.start() if self.cluster.cassandra_version() >= '2.2.0': session = self.patient_cql_connection(self.query_node) # Change system_auth keyspace replication factor to 2, otherwise replace will fail session.execute("""ALTER KEYSPACE system_auth WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};""") def _do_replace(self, same_address=False, jvm_option='replace_address', wait_other_notice=False, wait_for_binary_proto=True, replace_address=None, opts=None, data_center=None, extra_jvm_args=None): if replace_address is None: replace_address = self.replaced_node.address() # only create node if it's not yet created if self.replacement_node is None: replacement_address = '127.0.0.4' if same_address: replacement_address = self.replaced_node.address() self.cluster.remove(self.replaced_node) logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address)) self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True, thrift_interface=None, storage_interface=(replacement_address, 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042)) if opts is not None: logger.debug("Setting options on replacement node: {}".format(opts)) self.replacement_node.set_configuration_options(opts) self.cluster.add(self.replacement_node, False, data_center=data_center) if extra_jvm_args is None: extra_jvm_args = [] extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address), "-Dcassandra.ring_delay_ms=10000", "-Dcassandra.broadcast_interval_ms=10000"]) self.replacement_node.start(jvm_args=extra_jvm_args, wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice) if self.cluster.cassandra_version() >= '2.2.8' and same_address: self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement", timeout=60) def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE): if self.replaced_node.is_running(): logger.debug("Stopping {}".format(self.replaced_node.name)) self.replaced_node.stop(gently=gently, wait_other_notice=True) logger.debug("Testing node stoppage (query should fail).") with pytest.raises((Unavailable, ReadTimeout)): session = self.patient_cql_connection(self.query_node) query = SimpleStatement('select * from {}'.format(table), consistency_level=cl) session.execute(query) def _insert_data(self, n='1k', rf=3, whitelist=False): logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf)) self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf), '-rate', 'threads=10'], whitelist=whitelist) self.cluster.flush() time.sleep(20) def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000): logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit)) session = self.patient_cql_connection(self.query_node) query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl) return rows_to_list(session.execute(query, timeout=20)) def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000, restart_nodes=False): assert len(initial_data) > 0, "Initial data must be greater than 0" # query should work again logger.debug("Stopping old nodes") for node in self.cluster.nodelist(): if node.is_running() and node != self.replacement_node: logger.debug("Stopping {}".format(node.name)) node.stop(gently=False, wait_other_notice=True) logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit)) session = self.patient_exclusive_cql_connection(self.replacement_node) assert_all(session, 'select * from {} LIMIT {}'.format(table, limit), expected=initial_data, cl=cl) def _verify_replacement(self, node, same_address): if not same_address: if self.cluster.cassandra_version() >= '2.2.7': node.watch_log_for("Node {} is replacing {}" .format(self.replacement_node.address_for_current_version_slashy(), self.replaced_node.address_for_current_version_slashy()), timeout=60, filename='debug.log') node.watch_log_for("Node {} will complete replacement of {} for tokens" .format(self.replacement_node.address_for_current_version_slashy(), self.replaced_node.address_for_current_version_slashy()), timeout=10) node.watch_log_for("removing endpoint {}".format(self.replaced_node.address_for_current_version_slashy()), timeout=60, filename='debug.log') else: node.watch_log_for("between /{} and /{}; /{} is the new owner" .format(self.replaced_node.address(), self.replacement_node.address(), self.replacement_node.address()), timeout=60) def _verify_tokens_migrated_successfully(self, previous_log_size=None): if not self.dtest_config.use_vnodes: num_tokens = 1 else: # a little hacky but grep_log returns the whole line... num_tokens = int(self.replacement_node.get_conf_option('num_tokens')) logger.debug("Verifying {} tokens migrated successfully".format(num_tokens)) replmnt_address = self.replacement_node.address_for_current_version_slashy() repled_address = self.replaced_node.address_for_current_version_slashy() token_ownership_log = r"Token (.*?) changing ownership from {} to {}".format(repled_address, replmnt_address) logs = self.replacement_node.grep_log(token_ownership_log) if (previous_log_size is not None): assert len(logs) == previous_log_size moved_tokens = set([l[1].group(1) for l in logs]) logger.debug("number of moved tokens: {}".format(len(moved_tokens))) assert len(moved_tokens) == num_tokens return len(logs) def _test_insert_data_during_replace(self, same_address, mixed_versions=False): """ @jira_ticket CASSANDRA-8523 """ default_install_dir = self.cluster.get_install_dir() self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions) self._insert_data(n='1k') initial_data = self._fetch_initial_data() self._stop_node_to_replace() if mixed_versions: logger.debug("Upgrading all except {} to current version".format(self.query_node.address())) self.cluster.set_install_dir(install_dir=default_install_dir) for node in self.cluster.nodelist(): if node.is_running() and node != self.query_node: logger.debug("Upgrading {} to current version".format(node.address())) node.stop(gently=True, wait_other_notice=True) node.start(wait_for_binary_proto=True) # start node in current version on write survey mode self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"]) # Insert additional keys on query node self._insert_data(n='2k', whitelist=True) # If not same address or mixed versions, query node should forward writes to replacement node # so we update initial data to reflect additional keys inserted during replace if not same_address and not mixed_versions: initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO) logger.debug("Joining replaced node") self.replacement_node.nodetool("join") if not same_address: for node in self.cluster.nodelist(): # if mixed version, query node is not upgraded so it will not print replacement log if node.is_running() and (not mixed_versions or node != self.query_node): self._verify_replacement(node, same_address) self._verify_data(initial_data)
def replace_with_reset_resume_state_test(self): """Test replace with resetting bootstrap progress""" cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() node1.stress([ 'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)' ]) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE) initial_data = rows_to_list(session.execute(query)) node3.stop(gently=False) # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # replace node 3 with node 4 debug("Starting node 4 to replace node 3") node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160), storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=('127.0.0.4', 9042)) # keep timeout low so that test won't hang node4.set_configuration_options( values={'streaming_socket_timeout_in_ms': 1000}) cluster.add(node4, False) try: node4.start( jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False) except NodeError: pass # node doesn't start as expected t.join() node1.start() # restart node4 bootstrap with resetting bootstrap state node4.stop() mark = node4.mark_log() node4.start(jvm_args=[ "-Dcassandra.replace_address_first_boot=127.0.0.3", "-Dcassandra.reset_bootstrap_progress=true" ]) # check if we reset bootstrap state node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark) # wait for node3 ready to query node4.watch_log_for("Listening for thrift clients...", from_mark=mark) # check if 2nd bootstrap succeeded assert_bootstrap_state(self, node4, 'COMPLETED') # query should work again debug("Stopping old nodes") node1.stop(gently=False, wait_other_notice=True) node2.stop(gently=False, wait_other_notice=True) debug("Verifying data on new node.") session = self.patient_exclusive_cql_connection(node4) assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table), expected=initial_data, cl=ConsistencyLevel.ONE)
def resumable_replace_test(self): """ Test resumable bootstrap while replacing node. Feature introduced in 2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838 @jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838 """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() node1.stress([ 'write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)' ]) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE) initial_data = rows_to_list(session.execute(query)) node3.stop(gently=False) # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # replace node 3 with node 4 debug("Starting node 4 to replace node 3") node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160), storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=('127.0.0.4', 9042)) # keep timeout low so that test won't hang node4.set_configuration_options( values={'streaming_socket_timeout_in_ms': 1000}) cluster.add(node4, False) try: node4.start( jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False) except NodeError: pass # node doesn't start as expected t.join() # bring back node1 and invoke nodetool bootstrap to resume bootstrapping node1.start() node4.nodetool('bootstrap resume') # check if we skipped already retrieved ranges node4.watch_log_for("already available. Skipping streaming.") # wait for node3 ready to query node4.watch_log_for("Listening for thrift clients...") # check if 2nd bootstrap succeeded assert_bootstrap_state(self, node4, 'COMPLETED') # query should work again debug("Stopping old nodes") node1.stop(gently=False, wait_other_notice=True) node2.stop(gently=False, wait_other_notice=True) debug("Verifying data on new node.") session = self.patient_exclusive_cql_connection(node4) assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table), expected=initial_data, cl=ConsistencyLevel.ONE)
class BaseReplaceAddressTest(Tester): @pytest.fixture(autouse=True) def fixture_add_additional_log_patterns(self, fixture_dtest_setup): fixture_dtest_setup.ignore_log_patterns = ( # This one occurs when trying to send the migration to a # node that hasn't started yet, and when it does, it gets # replayed and everything is fine. r'Can\'t send migration request: node.*is down', r'Migration task failed to complete', # 10978 # ignore streaming error during bootstrap r'Streaming error occurred', r'failed stream session', r'Failed to properly handshake with peer' ) def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False): logger.debug("Starting cluster with {} nodes.".format(n)) self.cluster.populate(n) if opts is not None: logger.debug("Setting cluster options: {}".format(opts)) self.cluster.set_configuration_options(opts) self.cluster.set_batch_commitlog(enabled=True) self.query_node = self.cluster.nodelist()[0] self.replaced_node = self.cluster.nodelist()[-1] self.cluster.seeds.remove(self.replaced_node) NUM_TOKENS = os.environ.get('NUM_TOKENS', '256') if not self.dtest_config.use_vnodes: self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': 1}) else: self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': NUM_TOKENS}) if enable_byteman: # set up byteman self.query_node.byteman_port = '8100' self.query_node.import_config_files() if mixed_versions: logger.debug("Starting nodes on version 2.2.4") self.cluster.set_install_dir(version="2.2.4") self.cluster.start() if self.cluster.cassandra_version() >= '2.2.0': session = self.patient_cql_connection(self.query_node) # Change system_auth keyspace replication factor to 2, otherwise replace will fail session.execute("""ALTER KEYSPACE system_auth WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};""") def _do_replace(self, same_address=False, jvm_option='replace_address', wait_other_notice=False, wait_for_binary_proto=True, replace_address=None, opts=None, data_center=None, extra_jvm_args=None): if replace_address is None: replace_address = self.replaced_node.address() # only create node if it's not yet created if self.replacement_node is None: replacement_address = '127.0.0.4' if same_address: replacement_address = self.replaced_node.address() self.cluster.remove(self.replaced_node) logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address)) self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True, thrift_interface=None, storage_interface=(replacement_address, 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042)) if opts is not None: logger.debug("Setting options on replacement node: {}".format(opts)) self.replacement_node.set_configuration_options(opts) self.cluster.add(self.replacement_node, False, data_center=data_center) if extra_jvm_args is None: extra_jvm_args = [] extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address), "-Dcassandra.ring_delay_ms=10000", "-Dcassandra.broadcast_interval_ms=10000"]) self.replacement_node.start(jvm_args=extra_jvm_args, wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice) if self.cluster.cassandra_version() >= '2.2.8' and same_address: self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement", timeout=60) def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE): if self.replaced_node.is_running(): logger.debug("Stopping {}".format(self.replaced_node.name)) self.replaced_node.stop(gently=gently, wait_other_notice=True) logger.debug("Testing node stoppage (query should fail).") with pytest.raises((Unavailable, ReadTimeout)): session = self.patient_cql_connection(self.query_node) query = SimpleStatement('select * from {}'.format(table), consistency_level=cl) session.execute(query) def _insert_data(self, n='1k', rf=3, whitelist=False): logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf)) self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf), '-rate', 'threads=10'], whitelist=whitelist) self.cluster.flush() time.sleep(20) def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000): logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit)) session = self.patient_cql_connection(self.query_node) query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl) return rows_to_list(session.execute(query, timeout=20)) def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000, restart_nodes=False): assert len(initial_data) > 0, "Initial data must be greater than 0" # query should work again logger.debug("Stopping old nodes") for node in self.cluster.nodelist(): if node.is_running() and node != self.replacement_node: logger.debug("Stopping {}".format(node.name)) node.stop(gently=False, wait_other_notice=True) logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit)) session = self.patient_exclusive_cql_connection(self.replacement_node) assert_all(session, 'select * from {} LIMIT {}'.format(table, limit), expected=initial_data, cl=cl) def _verify_replacement(self, node, same_address): if not same_address: if self.cluster.cassandra_version() >= '2.2.7': address_prefix = '' if self.cluster.cassandra_version() >= '4.0' else '/' node.watch_log_for("Node {}{} is replacing {}{}" .format(address_prefix, self.replacement_node.address_for_current_version(), address_prefix, self.replaced_node.address_for_current_version()), timeout=60, filename='debug.log') node.watch_log_for("Node {}{} will complete replacement of {}{} for tokens" .format(address_prefix, self.replacement_node.address_for_current_version(), address_prefix, self.replaced_node.address_for_current_version()), timeout=10) node.watch_log_for("removing endpoint {}{}".format(address_prefix, self.replaced_node.address_for_current_version()), timeout=60, filename='debug.log') else: node.watch_log_for("between /{} and /{}; /{} is the new owner" .format(self.replaced_node.address(), self.replacement_node.address(), self.replacement_node.address()), timeout=60) def _verify_tokens_migrated_successfully(self, previous_log_size=None): if not self.dtest_config.use_vnodes: num_tokens = 1 else: # a little hacky but grep_log returns the whole line... num_tokens = int(self.replacement_node.get_conf_option('num_tokens')) logger.debug("Verifying {} tokens migrated sucessfully".format(num_tokens)) logs = self.replacement_node.grep_log(r"Token (.*?) changing ownership from /{} to /{}" .format(self.replaced_node.address(), self.replacement_node.address())) if (previous_log_size is not None): assert len(logs) == previous_log_size moved_tokens = set([l[1].group(1) for l in logs]) logger.debug("number of moved tokens: {}".format(len(moved_tokens))) assert len(moved_tokens) == num_tokens return len(logs) def _test_insert_data_during_replace(self, same_address, mixed_versions=False): """ @jira_ticket CASSANDRA-8523 """ default_install_dir = self.cluster.get_install_dir() self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions) self._insert_data(n='1k') initial_data = self._fetch_initial_data() self._stop_node_to_replace() if mixed_versions: logger.debug("Upgrading all except {} to current version".format(self.query_node.address())) self.cluster.set_install_dir(install_dir=default_install_dir) for node in self.cluster.nodelist(): if node.is_running() and node != self.query_node: logger.debug("Upgrading {} to current version".format(node.address())) node.stop(gently=True, wait_other_notice=True) node.start(wait_other_notice=True, wait_for_binary_proto=True) # start node in current version on write survey mode self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"]) # Insert additional keys on query node self._insert_data(n='2k', whitelist=True) # If not same address or mixed versions, query node should forward writes to replacement node # so we update initial data to reflect additional keys inserted during replace if not same_address and not mixed_versions: initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO) logger.debug("Joining replaced node") self.replacement_node.nodetool("join") if not same_address: for node in self.cluster.nodelist(): # if mixed version, query node is not upgraded so it will not print replacement log if node.is_running() and (not mixed_versions or node != self.query_node): self._verify_replacement(node, same_address) self._verify_data(initial_data)