class BaseReplaceAddressTest(Tester): @pytest.fixture(autouse=True) def fixture_add_additional_log_patterns(self, fixture_dtest_setup): fixture_dtest_setup.ignore_log_patterns = ( # This one occurs when trying to send the migration to a # node that hasn't started yet, and when it does, it gets # replayed and everything is fine. r'Can\'t send migration request: node.*is down', r'Migration task failed to complete', # 10978 # ignore streaming error during bootstrap r'Streaming error occurred', r'failed stream session', r'Failed to properly handshake with peer' ) def _setup(self, n=3, opts=None, enable_byteman=False, mixed_versions=False): logger.debug("Starting cluster with {} nodes.".format(n)) self.cluster.populate(n) if opts is not None: logger.debug("Setting cluster options: {}".format(opts)) self.cluster.set_configuration_options(opts) self.cluster.set_batch_commitlog(enabled=True) self.query_node = self.cluster.nodelist()[0] self.replaced_node = self.cluster.nodelist()[-1] self.cluster.seeds.remove(self.replaced_node) NUM_TOKENS = os.environ.get('NUM_TOKENS', '256') if not self.dtest_config.use_vnodes: self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': 1}) else: self.cluster.set_configuration_options(values={'initial_token': None, 'num_tokens': NUM_TOKENS}) if enable_byteman: # set up byteman self.query_node.byteman_port = '8100' self.query_node.import_config_files() if mixed_versions: logger.debug("Starting nodes on version 2.2.4") self.cluster.set_install_dir(version="2.2.4") self.cluster.start() if self.cluster.cassandra_version() >= '2.2.0': session = self.patient_cql_connection(self.query_node) # Change system_auth keyspace replication factor to 2, otherwise replace will fail session.execute("""ALTER KEYSPACE system_auth WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};""") def _do_replace(self, same_address=False, jvm_option='replace_address', wait_other_notice=False, wait_for_binary_proto=True, replace_address=None, opts=None, data_center=None, extra_jvm_args=None): if replace_address is None: replace_address = self.replaced_node.address() # only create node if it's not yet created if self.replacement_node is None: replacement_address = '127.0.0.4' if same_address: replacement_address = self.replaced_node.address() self.cluster.remove(self.replaced_node) logger.debug("Starting replacement node {} with jvm_option '{}={}'".format(replacement_address, jvm_option, replace_address)) self.replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True, thrift_interface=None, storage_interface=(replacement_address, 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042)) if opts is not None: logger.debug("Setting options on replacement node: {}".format(opts)) self.replacement_node.set_configuration_options(opts) self.cluster.add(self.replacement_node, False, data_center=data_center) if extra_jvm_args is None: extra_jvm_args = [] extra_jvm_args.extend(["-Dcassandra.{}={}".format(jvm_option, replace_address), "-Dcassandra.ring_delay_ms=10000", "-Dcassandra.broadcast_interval_ms=10000"]) self.replacement_node.start(jvm_args=extra_jvm_args, wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice) if self.cluster.cassandra_version() >= '2.2.8' and same_address: self.replacement_node.watch_log_for("Writes will not be forwarded to this node during replacement", timeout=60) def _stop_node_to_replace(self, gently=False, table='keyspace1.standard1', cl=ConsistencyLevel.THREE): if self.replaced_node.is_running(): logger.debug("Stopping {}".format(self.replaced_node.name)) self.replaced_node.stop(gently=gently, wait_other_notice=True) logger.debug("Testing node stoppage (query should fail).") with pytest.raises((Unavailable, ReadTimeout)): session = self.patient_cql_connection(self.query_node) query = SimpleStatement('select * from {}'.format(table), consistency_level=cl) session.execute(query) def _insert_data(self, n='1k', rf=3, whitelist=False): logger.debug("Inserting {} entries with rf={} with stress...".format(n, rf)) self.query_node.stress(['write', 'n={}'.format(n), 'no-warmup', '-schema', 'replication(factor={})'.format(rf), '-rate', 'threads=10'], whitelist=whitelist) self.cluster.flush() time.sleep(20) def _fetch_initial_data(self, table='keyspace1.standard1', cl=ConsistencyLevel.THREE, limit=10000): logger.debug("Fetching initial data from {} on {} with CL={} and LIMIT={}".format(table, self.query_node.name, cl, limit)) session = self.patient_cql_connection(self.query_node) query = SimpleStatement('select * from {} LIMIT {}'.format(table, limit), consistency_level=cl) return rows_to_list(session.execute(query, timeout=20)) def _verify_data(self, initial_data, table='keyspace1.standard1', cl=ConsistencyLevel.ONE, limit=10000, restart_nodes=False): assert len(initial_data) > 0, "Initial data must be greater than 0" # query should work again logger.debug("Stopping old nodes") for node in self.cluster.nodelist(): if node.is_running() and node != self.replacement_node: logger.debug("Stopping {}".format(node.name)) node.stop(gently=False, wait_other_notice=True) logger.debug("Verifying {} on {} with CL={} and LIMIT={}".format(table, self.replacement_node.address(), cl, limit)) session = self.patient_exclusive_cql_connection(self.replacement_node) assert_all(session, 'select * from {} LIMIT {}'.format(table, limit), expected=initial_data, cl=cl) def _verify_replacement(self, node, same_address): if not same_address: if self.cluster.cassandra_version() >= '2.2.7': address_prefix = '' if self.cluster.cassandra_version() >= '4.0' else '/' node.watch_log_for("Node {}{} is replacing {}{}" .format(address_prefix, self.replacement_node.address_for_current_version(), address_prefix, self.replaced_node.address_for_current_version()), timeout=60, filename='debug.log') node.watch_log_for("Node {}{} will complete replacement of {}{} for tokens" .format(address_prefix, self.replacement_node.address_for_current_version(), address_prefix, self.replaced_node.address_for_current_version()), timeout=10) node.watch_log_for("removing endpoint {}{}".format(address_prefix, self.replaced_node.address_for_current_version()), timeout=60, filename='debug.log') else: node.watch_log_for("between /{} and /{}; /{} is the new owner" .format(self.replaced_node.address(), self.replacement_node.address(), self.replacement_node.address()), timeout=60) def _verify_tokens_migrated_successfully(self, previous_log_size=None): if not self.dtest_config.use_vnodes: num_tokens = 1 else: # a little hacky but grep_log returns the whole line... num_tokens = int(self.replacement_node.get_conf_option('num_tokens')) logger.debug("Verifying {} tokens migrated successfully".format(num_tokens)) replmnt_address = ("/" + self.replacement_node.address()) if self.cluster.version() < '4.0' else self.replacement_node.address_and_port() repled_address = ("/" + self.replaced_node.address()) if self.cluster.version() < '4.0' else self.replaced_node.address_and_port() token_ownership_log = r"Token (.*?) changing ownership from {} to {}".format(repled_address, replmnt_address) logs = self.replacement_node.grep_log(token_ownership_log) if (previous_log_size is not None): assert len(logs) == previous_log_size moved_tokens = set([l[1].group(1) for l in logs]) logger.debug("number of moved tokens: {}".format(len(moved_tokens))) assert len(moved_tokens) == num_tokens return len(logs) def _test_insert_data_during_replace(self, same_address, mixed_versions=False): """ @jira_ticket CASSANDRA-8523 """ default_install_dir = self.cluster.get_install_dir() self._setup(opts={'hinted_handoff_enabled': False}, mixed_versions=mixed_versions) self._insert_data(n='1k') initial_data = self._fetch_initial_data() self._stop_node_to_replace() if mixed_versions: logger.debug("Upgrading all except {} to current version".format(self.query_node.address())) self.cluster.set_install_dir(install_dir=default_install_dir) for node in self.cluster.nodelist(): if node.is_running() and node != self.query_node: logger.debug("Upgrading {} to current version".format(node.address())) node.stop(gently=True, wait_other_notice=True) node.start(wait_other_notice=True, wait_for_binary_proto=True) # start node in current version on write survey mode self._do_replace(same_address=same_address, extra_jvm_args=["-Dcassandra.write_survey=true"]) # Insert additional keys on query node self._insert_data(n='2k', whitelist=True) # If not same address or mixed versions, query node should forward writes to replacement node # so we update initial data to reflect additional keys inserted during replace if not same_address and not mixed_versions: initial_data = self._fetch_initial_data(cl=ConsistencyLevel.TWO) logger.debug("Joining replaced node") self.replacement_node.nodetool("join") if not same_address: for node in self.cluster.nodelist(): # if mixed version, query node is not upgraded so it will not print replacement log if node.is_running() and (not mixed_versions or node != self.query_node): self._verify_replacement(node, same_address) self._verify_data(initial_data)