def test_normal_read_repair(self): """ test the normal case """ node1, node2, node3 = self.cluster.nodelist() assert isinstance(node1, Node) assert isinstance(node2, Node) assert isinstance(node3, Node) session = self.get_cql_connection(node1, timeout=2) session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)")) node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)") # re-enable writes node2.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')]) node2.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')]) coordinator = node2 # Stop reads on coordinator in order to make sure we do not go through # the messaging service for the local reads with StorageProxy(node2) as storage_proxy, stop_reads(coordinator): assert storage_proxy.blocking_read_repair == 0 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 0 session = self.get_cql_connection(coordinator) expected = [kcv(1, 0, 1), kcv(1, 1, 2)] results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1")) assert listify(results) == expected assert storage_proxy.blocking_read_repair == 1 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 0
def test_failed_read_repair(self): """ If none of the disagreeing nodes ack the repair mutation, the read should fail """ node1, node2, node3 = self.cluster.nodelist() assert isinstance(node1, Node) assert isinstance(node2, Node) assert isinstance(node3, Node) session = self.get_cql_connection(node1, timeout=2) session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)")) node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) node2.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')]) with raises(WriteTimeout): session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")) node2.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')]) session = self.get_cql_connection(node2) with StorageProxy(node2) as storage_proxy: assert storage_proxy.blocking_read_repair == 0 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 0 with raises(ReadTimeout): session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1")) assert storage_proxy.blocking_read_repair > 0 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write > 0
def test_speculative_write(self): """ if one node doesn't respond to a read repair mutation, it should be sent to the remaining node """ node1, node2, node3 = self.cluster.nodelist() assert isinstance(node1, Node) assert isinstance(node2, Node) assert isinstance(node3, Node) session = self.get_cql_connection(node1, timeout=2) session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)")) node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)") # re-enable writes on node 3, leave them off on node2 node2.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')]) node1.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')]) with StorageProxy(node1) as storage_proxy: assert storage_proxy.blocking_read_repair == 0 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 0 session = self.get_cql_connection(node1) expected = [kcv(1, 0, 1), kcv(1, 1, 2)] results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1")) assert listify(results) == expected assert storage_proxy.blocking_read_repair == 1 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 1
def test_counter_leader_with_partial_view(self): """ Test leader election with a starting node. Testing that nodes do not elect as mutation leader a node with a partial view on the cluster. Note that byteman rules can be syntax checked via the following command: sh ./bin/bytemancheck.sh -cp ~/path_to/apache-cassandra-3.0.14-SNAPSHOT.jar ~/path_to/rule.btm @jira_ticket CASSANDRA-13043 """ cluster = self.cluster cluster.populate(3, use_vnodes=True, install_byteman=True) nodes = cluster.nodelist() # Have node 1 and 3 cheat a bit during the leader election for a counter mutation; note that cheating # takes place iff there is an actual chance for node 2 to be picked. if cluster.version() < '4.0': nodes[0].update_startup_byteman_script( mk_bman_path('pre4.0/election_counter_leader_favor_node2.btm')) nodes[2].update_startup_byteman_script( mk_bman_path('pre4.0/election_counter_leader_favor_node2.btm')) else: nodes[0].update_startup_byteman_script( mk_bman_path('4.0/election_counter_leader_favor_node2.btm')) nodes[2].update_startup_byteman_script( mk_bman_path('4.0/election_counter_leader_favor_node2.btm')) cluster.start() session = self.patient_cql_connection(nodes[0]) create_ks(session, 'ks', 3) create_cf(session, 'cf', validation="CounterColumnType", columns={'c': 'counter'}) # Now stop the node and restart but first install a rule to slow down how fast node 2 will update the list # nodes that are alive nodes[1].stop(wait=True, wait_other_notice=False) nodes[1].update_startup_byteman_script( mk_bman_path('gossip_alive_callback_sleep.btm')) nodes[1].start(no_wait=True, wait_other_notice=False) # Until node 2 is fully alive try to force other nodes to pick him as mutation leader. # If CASSANDRA-13043 is fixed, they will not. Otherwise they will do, but since we are slowing down how # fast node 2 updates the list of nodes that are alive, it will just have a partial view on the cluster # and thus will raise an 'UnavailableException' exception. nb_attempts = 50000 for i in range(0, nb_attempts): # Change the name of the counter for the sake of randomization q = SimpleStatement( query_string= "UPDATE ks.cf SET c = c + 1 WHERE key = 'counter_%d'" % i, consistency_level=ConsistencyLevel.QUORUM) session.execute(q)
def test_bootstrap_waits_for_streaming_to_finish(self): """ Test that bootstrap completes and is marked as such after streaming finishes. """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') logger.debug("Create a cluster") cluster.populate(1) node1 = cluster.nodelist()[0] logger.debug("Start node 1") node1.start(wait_for_binary_proto=True) logger.debug("Insert 10k rows") node1.stress([ 'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema', 'replication(factor=2)' ]) logger.debug("Bootstrap node 2 with delay") node2 = new_node(cluster, byteman_port='4200') node2.update_startup_byteman_script( mk_bman_path('bootstrap_5s_sleep.btm')) node2.start(wait_for_binary_proto=True) assert_bootstrap_state(self, node2, 'COMPLETED') assert node2.grep_log('Bootstrap completed', filename='debug.log')
def test_transient_full_merge_read(self): """ When reading, transient replica should serve a missing read """ for node in self.nodes: self.assert_has_no_sstables(node) tm = lambda n: self.table_metrics(n) self.insert_row(1, 1, 1) # Stop writes to the other full node self.node2.byteman_submit([mk_bman_path('stop_writes.btm')]) self.insert_row(1, 2, 2) self.assert_local_rows(self.node1, [[1, 1, 1], [1, 2, 2]]) self.assert_local_rows(self.node2, [[1, 1, 1]]) self.assert_local_rows(self.node3, [[1, 1, 1], [1, 2, 2]]) self.assert_local_rows(self.node4, [[1, 2, 2]]) self.assert_local_rows(self.node5, [[1, 2, 2]]) # Stop reads from the node that will hold the second row self.node1.stop() # Whether we're reading from the full node or from the transient node, we should get consistent results for node in [self.node2, self.node3, self.node4, self.node5]: assert_all(self.exclusive_cql_connection(node), "SELECT * FROM %s.%s" % (self.keyspace, self.table), [[1, 1, 1], [1, 2, 2]], cl=ConsistencyLevel.QUORUM)
def _batchlog_replay_compatibility_test(self, coordinator_idx, current_nodes, previous_version, previous_nodes, protocol_version): session = self.prepare_mixed(coordinator_idx, current_nodes, previous_version, previous_nodes, protocol_version=protocol_version, install_byteman=True) coordinator = self.cluster.nodelist()[coordinator_idx] coordinator.byteman_submit( [mk_bman_path('fail_after_batchlog_write.btm')]) logger.debug( "Injected byteman scripts to enable batchlog replay {}".format( coordinator.name)) query = """ BEGIN BATCH INSERT INTO users (id, firstname, lastname) VALUES (0, 'Jack', 'Sparrow') INSERT INTO users (id, firstname, lastname) VALUES (1, 'Will', 'Turner') APPLY BATCH """ session.execute(query) # batchlog replay skips over all entries that are younger than # 2 * write_request_timeout_in_ms ms: 1x timeout for all mutations to be written, # and another 1x timeout for batch remove mutation to be received. delay = 2 * coordinator.get_conf_option( 'write_request_timeout_in_ms') / 1000.0 + 1 logger.debug( 'Sleeping for {}s for the batches to not be skipped'.format(delay)) time.sleep(delay) total_batches_replayed = 0 blm = make_mbean('db', type='BatchlogManager') for n in self.cluster.nodelist(): if n == coordinator: continue with JolokiaAgent(n) as jmx: logger.debug('Forcing batchlog replay for {}'.format(n.name)) jmx.execute_method(blm, 'forceBatchlogReplay') batches_replayed = jmx.read_attribute(blm, 'TotalBatchesReplayed') logger.debug('{} batches replayed on node {}'.format( batches_replayed, n.name)) total_batches_replayed += batches_replayed assert total_batches_replayed >= 2 for node in self.cluster.nodelist(): session = self.patient_exclusive_cql_connection( node, protocol_version=protocol_version) rows = sorted( session.execute( 'SELECT id, firstname, lastname FROM ks.users')) assert [[0, 'Jack', 'Sparrow'], [1, 'Will', 'Turner']], [list(rows[0]) == list(rows[1])]
def test_speculative_write(self): """ if a full replica isn't responding, we should send the write to the transient replica """ session = self.exclusive_cql_connection(self.node1) self.node2.byteman_submit([mk_bman_path('slow_writes.btm')]) self.insert_row(1, 1, 1, session=session) self.assert_local_rows(self.node1, [[1, 1, 1]]) self.assert_local_rows(self.node2, []) self.assert_local_rows(self.node3, [[1, 1, 1]])
def fixture_set_cluster_settings(self, fixture_dtest_setup): cluster = fixture_dtest_setup.cluster cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'dynamic_snitch': False, 'write_request_timeout_in_ms': 1000, 'read_request_timeout_in_ms': 1000}) cluster.populate(3, install_byteman=True, debug=True) byteman_validate(cluster.nodelist()[0], mk_bman_path('read_repair/sorted_live_endpoints.btm'), verbose=True) cluster.start(jvm_args=['-XX:-PerfDisableSharedMem']) session = fixture_dtest_setup.patient_exclusive_cql_connection(cluster.nodelist()[0], timeout=2) session.execute("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c)) WITH speculative_retry = '400ms';")
def test_simple_bootstrap_small_keepalive_period(self): """ @jira_ticket CASSANDRA-11841 Test that bootstrap completes if it takes longer than streaming_socket_timeout_in_ms or 2*streaming_keep_alive_period_in_secs to receive a single sstable """ cluster = self.cluster yaml_opts = {'streaming_keep_alive_period_in_secs': 2} if cluster.version() < '4.0': yaml_opts['streaming_socket_timeout_in_ms'] = 1000 cluster.set_configuration_options(values=yaml_opts) # Create a single node cluster cluster.populate(1) node1 = cluster.nodelist()[0] logger.debug("Setting up byteman on {}".format(node1.name)) # set up byteman node1.byteman_port = '8100' node1.import_config_files() cluster.start() # Create more than one sstable larger than 1MB node1.stress([ 'write', 'n=1K', '-rate', 'threads=8', '-schema', 'compaction(strategy=SizeTieredCompactionStrategy, enabled=false)' ]) cluster.flush() logger.debug("Submitting byteman script to {} to".format(node1.name)) # Sleep longer than streaming_socket_timeout_in_ms to make sure the node will not be killed node1.byteman_submit([mk_bman_path('stream_5s_sleep.btm')]) # Bootstraping a new node with very small streaming_socket_timeout_in_ms node2 = new_node(cluster) node2.start(wait_for_binary_proto=True) # Shouldn't fail due to streaming socket timeout timeout assert_bootstrap_state(self, node2, 'COMPLETED') if cluster.version() < '4.0': for node in cluster.nodelist(): assert node.grep_log( 'Scheduling keep-alive task with 2s period.', filename='debug.log') assert node.grep_log('Sending keep-alive', filename='debug.log') assert node.grep_log('Received keep-alive', filename='debug.log')
def _test_speculative_write_repair_cycle(self, primary_range, optimized_repair, repair_coordinator, expect_node3_data, use_lcs=False): """ if one of the full replicas is not available, data should be written to the transient replica, but removed after incremental repair """ for node in self.nodes: self.assert_has_no_sstables(node) if use_lcs: self.use_lcs() self.node2.byteman_submit([mk_bman_path('stop_writes.btm')]) # self.insert_row(1) tm = lambda n: self.table_metrics(n) with tm(self.node1) as tm1, tm(self.node2) as tm2, tm( self.node3) as tm3: assert tm1.write_count == 0 assert tm2.write_count == 0 assert tm3.write_count == 0 self.insert_row(1, 1, 1) assert tm1.write_count == 1 assert tm2.write_count == 0 assert tm3.write_count == 1 self.assert_has_sstables(self.node1, flush=True) self.assert_has_no_sstables(self.node2, flush=True) self.assert_has_sstables(self.node3, flush=True) repair_opts = ['repair', self.keyspace] if primary_range: repair_opts.append('-pr') if optimized_repair: repair_opts.append('-os') self.node1.nodetool(' '.join(repair_opts)) self.assert_has_sstables(self.node1, compact=True) self.assert_has_sstables(self.node2, compact=True) if expect_node3_data: self.assert_has_sstables(self.node3, compact=True) else: self.assert_has_no_sstables(self.node3, compact=True) entire_sstable = "true" if self.stream_entire_sstables() else "false" assert self.node2.grep_log( 'Incoming stream entireSSTable={}'.format(entire_sstable), filename='debug.log')
def test_custom_speculate(self): """ If write can't succeed on full replica, it's written to the transient node instead """ session = self.exclusive_cql_connection(self.node1) session.execute( "ALTER TABLE %s.%s WITH speculative_retry = '99.99PERCENTILE';" % (self.keyspace, self.table)) self.insert_row(1, 1, 1) # Stop writes to the other full node self.node2.byteman_submit([mk_bman_path('stop_writes.btm')]) self.insert_row(1, 2, 2) for node in self.nodes: assert_all(self.exclusive_cql_connection(node), "SELECT * FROM %s.%s WHERE pk = 1" % (self.keyspace, self.table), [[1, 1, 1], [1, 2, 2]], cl=ConsistencyLevel.QUORUM)
def test_quorum_requirement_on_speculated_read(self): """ Even if we speculate on every stage, we should still only require a quorum of responses for success """ node1, node2, node3 = self.cluster.nodelist() assert isinstance(node1, Node) assert isinstance(node2, Node) assert isinstance(node3, Node) session = self.get_cql_connection(node1, timeout=2) session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)")) node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)") # re-enable writes node2.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')]) node3.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')]) # force endpoint order node1.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')]) node2.byteman_submit([mk_bman_path('read_repair/stop_digest_reads.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_data_reads.btm')]) node2.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')]) with StorageProxy(node1) as storage_proxy: assert storage_proxy.get_table_metric("ks", "tbl", "SpeculativeRetries") == 0 assert storage_proxy.blocking_read_repair == 0 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 0 session = self.get_cql_connection(node1) expected = [kcv(1, 0, 1), kcv(1, 1, 2)] results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1")) assert listify(results) == expected assert storage_proxy.get_table_metric("ks", "tbl", "SpeculativeRetries") == 1 assert storage_proxy.blocking_read_repair == 1 assert storage_proxy.speculated_rr_read == 0 # there shouldn't be any replicas to speculate on assert storage_proxy.speculated_rr_write == 1
def _test_missed_update_with_transient_replicas(self, missed_by_transient): cluster = self.cluster cluster.set_configuration_options( values={ 'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500, 'enable_transient_replication': True, 'partitioner': 'org.apache.cassandra.dht.OrderPreservingPartitioner' }) cluster.set_batch_commitlog(enabled=True) cluster.populate(2, tokens=[0, 1], debug=True, install_byteman=True) node1, node2 = cluster.nodelist() cluster.start() self.session = self.patient_exclusive_cql_connection( node1, consistency_level=CL.ALL) self.session.execute( "CREATE KEYSPACE %s WITH replication = " "{'class': 'SimpleStrategy', 'replication_factor': '2/1'}" % (keyspace)) self.session.execute("USE " + keyspace) self.session.execute("CREATE TABLE t (k int PRIMARY KEY, v text)" " WITH speculative_retry = 'NEVER'" " AND additional_write_policy = 'NEVER'" " AND read_repair = 'NONE'") # insert in both nodes with CL=ALL self.session.execute("INSERT INTO t(k, v) VALUES (0, 'old')") # update the previous value with CL=ONE only in one replica node = cluster.nodelist()[1 if missed_by_transient else 0] node.byteman_submit([mk_bman_path('stop_writes.btm')]) self.session.execute( SimpleStatement("UPDATE t SET v = 'new' WHERE k = 0", consistency_level=CL.ONE)) # query with CL=ALL to verify that no old values are resurrected self._assert_none("SELECT * FROM t WHERE v = 'old'") self._assert_one("SELECT * FROM t WHERE v = 'new'", row=[0, 'new'])
def test_speculative_data_request(self): """ If one node doesn't respond to a full data request, it should query the other """ node1, node2, node3 = self.cluster.nodelist() assert isinstance(node1, Node) assert isinstance(node2, Node) assert isinstance(node3, Node) session = self.get_cql_connection(node1, timeout=2) session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)")) node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')]) session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)") # re-enable writes node2.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')]) node1.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')]) version = self.cluster.cassandra_version() if version < '4.1': node1.byteman_submit([mk_bman_path('request_verb_timing.btm')]) else: node1.byteman_submit([mk_bman_path('post4.0/request_verb_timing.btm')]) with StorageProxy(node1) as storage_proxy: assert storage_proxy.blocking_read_repair == 0 assert storage_proxy.speculated_rr_read == 0 assert storage_proxy.speculated_rr_write == 0 session = self.get_cql_connection(node1) node2.byteman_submit([mk_bman_path('read_repair/stop_data_reads.btm')]) results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1")) timing = request_verb_timing(node1) repair_req_node3 = timing[node3.ip_addr].get('READ_REPAIR_REQ') repair_req_node2 = timing[node2.ip_addr].get('READ_REPAIR_REQ') assert listify(results) == [kcv(1, 0, 1), kcv(1, 1, 2)] assert storage_proxy.blocking_read_repair == 1 assert storage_proxy.speculated_rr_read == 1 # under normal circumstances we don't expect a speculated write here, # but the repair request to node 3 may timeout due to CPU contention and # then a speculated write is sent to node 2, so we just make sure that the # request to node 2 didn't happen before the request to node 3 assert storage_proxy.speculated_rr_write == 0 or repair_req_node2 > repair_req_node3
def _byteman_cycle(nodes, scripts): script_path = lambda name: mk_bman_path('read_repair/') + name + '.btm' for script in scripts: byteman_validate(nodes[0], script_path(script)) for node in nodes: assert isinstance(node, Node) for name in scripts: print(node.name) node.byteman_submit([script_path(name)]) yield for node in nodes: for name in scripts: print(node.name) node.byteman_submit(['-u', script_path(name)])
def _test_transient_full_merge_read_with_delete(self, coordinator): """ When reading, transient replica should serve a missing read """ for node in self.nodes: self.assert_has_no_sstables(node) tm = lambda n: self.table_metrics(n) self.insert_row(1, 1, 1) self.insert_row(1, 2, 2) # Stop writes to the other full node self.node2.byteman_submit([mk_bman_path('stop_writes.btm')]) self.delete_row(1, 2) self.assert_local_rows(self.node3, []) # Stop reads from the node that will hold the second row self.node1.stop() assert_all(self.exclusive_cql_connection(coordinator), "SELECT * FROM %s.%s" % (self.keyspace, self.table), [[1, 1, 1]], cl=ConsistencyLevel.QUORUM)
def test_transient_write(self): """ If write can't succeed on full replica, it's written to the transient node instead """ for node in self.nodes: self.assert_has_no_sstables(node) tm = lambda n: self.table_metrics(n) with tm(self.node1) as tm1, tm(self.node2) as tm2, tm( self.node3) as tm3: self.insert_row(1, 1, 1) # Stop writes to the other full node self.node2.byteman_submit([mk_bman_path('stop_writes.btm')]) self.insert_row(1, 2, 2) # node1 should contain both rows self.assert_local_rows(self.node1, [[1, 1, 1], [1, 2, 2]]) # write couldn't succeed on node2, so it has only the first row self.assert_local_rows(self.node2, [[1, 1, 1]]) # transient replica should hold only the second row self.assert_local_rows(self.node3, [[1, 2, 2]])
def test_sstableloader_with_failing_2i(self): """ @jira_ticket CASSANDRA-10130 Simulates an index building failure during SSTables load. The table data should be loaded and the index should be marked for rebuilding during the next node start. """ def create_schema_with_2i(session): create_ks(session, 'k', 1) session.execute( "CREATE TABLE k.t (p int, c int, v int, PRIMARY KEY(p, c))") session.execute("CREATE INDEX idx ON k.t(v)") cluster = self.cluster cluster.populate(1, install_byteman=True).start() node = cluster.nodelist()[0] session = self.patient_cql_connection(node) create_schema_with_2i(session) session.execute("INSERT INTO k.t(p, c, v) VALUES (0, 1, 8)") # Stop node and copy SSTables node.nodetool('drain') node.stop() self.copy_sstables(cluster, node) # Wipe out data and restart cluster.clear() cluster.start() # Restore the schema session = self.patient_cql_connection(node) create_schema_with_2i(session) # The table should exist and be empty, and the index should be empty and marked as built assert_one( session, """SELECT * FROM system."IndexInfo" WHERE table_name='k'""", ['k', 'idx', None]) assert_none(session, "SELECT * FROM k.t") assert_none(session, "SELECT * FROM k.t WHERE v = 8") # Add some additional data before loading the SSTable, to check that it will be still accessible session.execute("INSERT INTO k.t(p, c, v) VALUES (0, 2, 8)") assert_one(session, "SELECT * FROM k.t", [0, 2, 8]) assert_one(session, "SELECT * FROM k.t WHERE v = 8", [0, 2, 8]) # Load SSTables with a failure during index creation node.byteman_submit([mk_bman_path('index_build_failure.btm')]) with pytest.raises(Exception): self.load_sstables(cluster, node, 'k') # Check that the index isn't marked as built and the old SSTable data has been loaded but not indexed assert_none( session, """SELECT * FROM system."IndexInfo" WHERE table_name='k'""") assert_all(session, "SELECT * FROM k.t", [[0, 1, 8], [0, 2, 8]]) assert_one(session, "SELECT * FROM k.t WHERE v = 8", [0, 2, 8]) # Restart the node to trigger index rebuild node.nodetool('drain') node.stop() cluster.start() session = self.patient_cql_connection(node) # Check that the index is marked as built and the index has been rebuilt assert_one( session, """SELECT * FROM system."IndexInfo" WHERE table_name='k'""", ['k', 'idx', None]) assert_all(session, "SELECT * FROM k.t", [[0, 1, 8], [0, 2, 8]]) assert_all(session, "SELECT * FROM k.t WHERE v = 8", [[0, 1, 8], [0, 2, 8]])
class BootstrapTester(Tester): byteman_submit_path_pre_4_0 = mk_bman_path('pre4.0/stream_failure.btm') byteman_submit_path_4_0 = mk_bman_path('4.0/stream_failure.btm') @pytest.fixture(autouse=True) def fixture_add_additional_log_patterns(self, fixture_dtest_setup): fixture_dtest_setup.allow_log_errors = True fixture_dtest_setup.ignore_log_patterns = ( # This one occurs when trying to send the migration to a # node that hasn't started yet, and when it does, it gets # replayed and everything is fine. r'Can\'t send migration request: node.*is down', # ignore streaming error during bootstrap r'Exception encountered during startup', r'Streaming error occurred') def _base_bootstrap_test(self, bootstrap=None, bootstrap_from_version=None, enable_ssl=None): def default_bootstrap(cluster, token): node2 = new_node(cluster) node2.set_configuration_options(values={'initial_token': token}) node2.start(wait_for_binary_proto=True) return node2 if bootstrap is None: bootstrap = default_bootstrap cluster = self.cluster if enable_ssl: logger.debug("***using internode ssl***") generate_ssl_stores(self.fixture_dtest_setup.test_path) cluster.enable_internode_ssl(self.fixture_dtest_setup.test_path) tokens = cluster.balanced_tokens(2) cluster.set_configuration_options(values={'num_tokens': 1}) logger.debug("[node1, node2] tokens: %r" % (tokens, )) keys = 10000 # Create a single node cluster cluster.populate(1) node1 = cluster.nodelist()[0] if bootstrap_from_version: logger.debug("starting source node on version {}".format( bootstrap_from_version)) node1.set_install_dir(version=bootstrap_from_version) node1.set_configuration_options(values={'initial_token': tokens[0]}) cluster.start() session = self.patient_cql_connection(node1) create_ks(session, 'ks', 1) create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) # record the size before inserting any of our own data empty_size = data_size(node1, 'ks', 'cf') logger.debug("node1 empty size for ks.cf: %s" % float(empty_size)) insert_statement = session.prepare( "INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')") execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)]) node1.flush() node1.compact() initial_size = data_size(node1, 'ks', 'cf') logger.debug("node1 size for ks.cf before bootstrapping node2: %s" % float(initial_size)) # Reads inserted data all during the bootstrap process. We shouldn't # get any error query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE) session.shutdown() # Bootstrapping a new node in the current version node2 = bootstrap(cluster, tokens[1]) node2.compact() node1.cleanup() logger.debug("node1 size for ks.cf after cleanup: %s" % float(data_size(node1, 'ks', 'cf'))) node1.compact() logger.debug("node1 size for ks.cf after compacting: %s" % float(data_size(node1, 'ks', 'cf'))) logger.debug("node2 size for ks.cf after compacting: %s" % float(data_size(node2, 'ks', 'cf'))) size1 = float(data_size(node1, 'ks', 'cf')) size2 = float(data_size(node2, 'ks', 'cf')) assert_almost_equal(size1, size2, error=0.3) assert_almost_equal(float(initial_size - empty_size), 2 * (size1 - float(empty_size))) assert_bootstrap_state(self, node2, 'COMPLETED') @pytest.mark.no_vnodes def test_simple_bootstrap_with_ssl(self): self._base_bootstrap_test(enable_ssl=True) @pytest.mark.no_vnodes def test_simple_bootstrap(self): self._base_bootstrap_test() @pytest.mark.no_vnodes def test_bootstrap_on_write_survey(self): def bootstrap_on_write_survey_and_join(cluster, token): node2 = new_node(cluster) node2.set_configuration_options(values={'initial_token': token}) node2.start(jvm_args=["-Dcassandra.write_survey=true"], wait_for_binary_proto=True) assert len( node2.grep_log( 'Startup complete, but write survey mode is active, not becoming an active ring member.' )) assert_bootstrap_state(self, node2, 'IN_PROGRESS') node2.nodetool("join") assert len( node2.grep_log( 'Leaving write survey mode and joining ring at operator request' )) return node2 self._base_bootstrap_test(bootstrap_on_write_survey_and_join) def _test_bootstrap_with_compatibility_flag_on(self, bootstrap_from_version): def bootstrap_with_compatibility_flag_on(cluster, token): node2 = new_node(cluster) node2.set_configuration_options(values={'initial_token': token}) # cassandra.force_3_0_protocol_version parameter is needed to allow schema # changes during the bootstrapping for upgrades from 3.0.14+ to anything upwards for 3.0.x or 3.x clusters. # @jira_ticket CASSANDRA-13004 for detailed context on `cassandra.force_3_0_protocol_version` flag node2.start( jvm_args=["-Dcassandra.force_3_0_protocol_version=true"], wait_for_binary_proto=True) return node2 self._base_bootstrap_test( bootstrap_with_compatibility_flag_on, bootstrap_from_version=bootstrap_from_version) @since('3.10') @pytest.mark.no_vnodes def test_simple_bootstrap_small_keepalive_period(self): """ @jira_ticket CASSANDRA-11841 Test that bootstrap completes if it takes longer than streaming_socket_timeout_in_ms or 2*streaming_keep_alive_period_in_secs to receive a single sstable """ cluster = self.cluster yaml_opts = {'streaming_keep_alive_period_in_secs': 2} if cluster.version() < '4.0': yaml_opts['streaming_socket_timeout_in_ms'] = 1000 cluster.set_configuration_options(values=yaml_opts) # Create a single node cluster cluster.populate(1) node1 = cluster.nodelist()[0] logger.debug("Setting up byteman on {}".format(node1.name)) # set up byteman node1.byteman_port = '8100' node1.import_config_files() cluster.start() # Create more than one sstable larger than 1MB node1.stress([ 'write', 'n=1K', '-rate', 'threads=8', '-schema', 'compaction(strategy=SizeTieredCompactionStrategy, enabled=false)' ]) cluster.flush() logger.debug("Submitting byteman script to {} to".format(node1.name)) # Sleep longer than streaming_socket_timeout_in_ms to make sure the node will not be killed node1.byteman_submit([mk_bman_path('stream_5s_sleep.btm')]) # Bootstraping a new node with very small streaming_socket_timeout_in_ms node2 = new_node(cluster) node2.start(wait_for_binary_proto=True) # Shouldn't fail due to streaming socket timeout timeout assert_bootstrap_state(self, node2, 'COMPLETED') if cluster.version() < '4.0': for node in cluster.nodelist(): assert node.grep_log( 'Scheduling keep-alive task with 2s period.', filename='debug.log') assert node.grep_log('Sending keep-alive', filename='debug.log') assert node.grep_log('Received keep-alive', filename='debug.log') def test_simple_bootstrap_nodata(self): """ @jira_ticket CASSANDRA-11010 Test that bootstrap completes if streaming from nodes with no data """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') # Create a two-node cluster cluster.populate(2) cluster.start() # Bootstrapping a new node node3 = new_node(cluster) node3.start(wait_for_binary_proto=True) assert_bootstrap_state(self, node3, 'COMPLETED') def test_schema_removed_nodes(self): """ @jira_ticket CASSANDRA-16577 Test that nodes can bootstrap after a schema change performed with a node removed """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(2) cluster.start() node1, node2 = cluster.nodelist() node2.decommission(force=cluster.version() > '4') session = self.patient_cql_connection(node1) session.execute( "CREATE KEYSPACE k WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};" ) node3 = new_node(cluster) node3.start(wait_for_binary_proto=True) def test_read_from_bootstrapped_node(self): """ Test bootstrapped node sees existing data @jira_ticket CASSANDRA-6648 """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(3) cluster.start() node1 = cluster.nodes['node1'] node1.stress([ 'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema', 'replication(factor=2)' ]) session = self.patient_cql_connection(node1) stress_table = 'keyspace1.standard1' original_rows = list( session.execute("SELECT * FROM %s" % (stress_table, ))) node4 = new_node(cluster) node4.start(wait_for_binary_proto=True) session = self.patient_exclusive_cql_connection(node4) new_rows = list(session.execute("SELECT * FROM %s" % (stress_table, ))) assert original_rows == new_rows @since('3.0') def test_bootstrap_waits_for_streaming_to_finish(self): """ Test that bootstrap completes and is marked as such after streaming finishes. """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') logger.debug("Create a cluster") cluster.populate(1) node1 = cluster.nodelist()[0] logger.debug("Start node 1") node1.start(wait_for_binary_proto=True) logger.debug("Insert 10k rows") node1.stress([ 'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema', 'replication(factor=2)' ]) logger.debug("Bootstrap node 2 with delay") node2 = new_node(cluster, byteman_port='4200') node2.update_startup_byteman_script( mk_bman_path('bootstrap_5s_sleep.btm')) node2.start(wait_for_binary_proto=True) assert_bootstrap_state(self, node2, 'COMPLETED') assert node2.grep_log('Bootstrap completed', filename='debug.log') def test_consistent_range_movement_true_with_replica_down_should_fail( self): self._bootstrap_test_with_replica_down(True) def test_consistent_range_movement_false_with_replica_down_should_succeed( self): self._bootstrap_test_with_replica_down(False) def test_consistent_range_movement_true_with_rf1_should_fail(self): self._bootstrap_test_with_replica_down(True, rf=1) def test_consistent_range_movement_false_with_rf1_should_succeed(self): self._bootstrap_test_with_replica_down(False, rf=1) def test_rf_gt_nodes_multidc_should_succeed(self): """ Validating a KS with RF > N on multi DC doesn't break bootstrap @jira_ticket CASSANDRA-16296 CASSANDRA-16411 """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate([1, 1]) cluster.start() node1 = cluster.nodelist()[0] node2 = cluster.nodelist()[1] session = self.patient_exclusive_cql_connection(node1) session.execute( "CREATE KEYSPACE k WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'dc1' : '3'}" ) if cluster.version() >= '4.0': warning = 'Your replication factor 3 for keyspace k is higher than the number of nodes 1 for datacenter dc1' assert len(node1.grep_log(warning)) == 1 assert len(node2.grep_log(warning)) == 0 session.execute( "ALTER KEYSPACE k WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'dc1' : '2'}" ) session.execute( "CREATE TABLE k.testgtrfmultidc (KEY text PRIMARY KEY)") session.execute( "INSERT INTO k.testgtrfmultidc (KEY) VALUES ('test_rf_gt_nodes_multidc_should_succeed')" ) if cluster.version() >= '4.0': warning = 'Your replication factor 2 for keyspace k is higher than the number of nodes 1 for datacenter dc1' assert len(node1.grep_log(warning)) == 1 assert len(node2.grep_log(warning)) == 0 marks = map(lambda n: n.mark_log(), cluster.nodelist()) node3 = Node(name='node3', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.3', 9160), storage_interface=('127.0.0.3', 7000), jmx_port='7300', remote_debug_port='0', initial_token=None, binary_interface=('127.0.0.3', 9042)) cluster.add(node3, is_seed=False, data_center="dc1") node3.start(wait_for_binary_proto=True) if cluster.version() >= '4.0': warning = 'is higher than the number of nodes' for (node, mark) in zip(cluster.nodelist(), marks): assert len(node.grep_log(warning, from_mark=mark)) == 0 session3 = self.patient_exclusive_cql_connection(node3) assert_one(session3, "SELECT * FROM k.testgtrfmultidc", ["test_rf_gt_nodes_multidc_should_succeed"]) def _bootstrap_test_with_replica_down(self, consistent_range_movement, rf=2): """ Test to check consistent bootstrap will not succeed when there are insufficient replicas @jira_ticket CASSANDRA-11848 """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(2) node1, node2 = cluster.nodelist() node3_token = None # Make token assignment deterministic if not self.dtest_config.use_vnodes: cluster.set_configuration_options(values={'num_tokens': 1}) tokens = cluster.balanced_tokens(3) logger.debug("non-vnode tokens: %r" % (tokens, )) node1.set_configuration_options( values={'initial_token': tokens[0]}) node2.set_configuration_options( values={'initial_token': tokens[2]}) node3_token = tokens[1] # Add node 3 between node1 and node2 cluster.start() node1.stress([ 'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema', 'replication(factor={})'.format(rf) ]) # change system_auth keyspace to 2 (default is 1) to avoid # "Unable to find sufficient sources for streaming" warning if cluster.cassandra_version() >= '2.2.0': session = self.patient_cql_connection(node1) session.execute(""" ALTER KEYSPACE system_auth WITH replication = {'class':'SimpleStrategy', 'replication_factor':2}; """) # Stop node2, so node3 will not be able to perform consistent range movement node2.stop(wait_other_notice=True) successful_bootstrap_expected = not consistent_range_movement node3 = new_node(cluster, token=node3_token) node3.start(wait_for_binary_proto=successful_bootstrap_expected, wait_other_notice=successful_bootstrap_expected, jvm_args=[ "-Dcassandra.consistent.rangemovement={}".format( consistent_range_movement) ]) if successful_bootstrap_expected: # with rf=1 and cassandra.consistent.rangemovement=false, missing sources are ignored if not consistent_range_movement and rf == 1: node3.watch_log_for( "Unable to find sufficient sources for streaming range") assert node3.is_running() assert_bootstrap_state(self, node3, 'COMPLETED') else: if consistent_range_movement: if cluster.version() < '4.0': node3.watch_log_for( "A node required to move the data consistently is down" ) else: node3.watch_log_for( "Necessary replicas for strict consistency were removed by source filters" ) else: node3.watch_log_for( "Unable to find sufficient sources for streaming range") assert_not_running(node3) @since('2.2') def test_resumable_bootstrap(self): """ Test resuming bootstrap after data streaming failure """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(2) node1 = cluster.nodes['node1'] # set up byteman node1.byteman_port = '8100' node1.import_config_files() cluster.start() # kill stream to node3 in the middle of streaming to let it fail if cluster.version() < '4.0': node1.byteman_submit([self.byteman_submit_path_pre_4_0]) else: node1.byteman_submit([self.byteman_submit_path_4_0]) node1.stress([ 'write', 'n=1K', 'no-warmup', 'cl=TWO', '-schema', 'replication(factor=2)', '-rate', 'threads=50' ]) cluster.flush() # start bootstrapping node3 and wait for streaming node3 = new_node(cluster) node3.start(wait_other_notice=False) # let streaming fail as we expect node3.watch_log_for('Some data streaming failed') # bring back node3 and invoke nodetool bootstrap to resume bootstrapping node3.nodetool('bootstrap resume') node3.wait_for_binary_interface() assert_bootstrap_state(self, node3, 'COMPLETED') # cleanup to guarantee each node will only have sstables of its ranges cluster.cleanup() logger.debug("Check data is present") # Let's check stream bootstrap completely transferred data stdout, stderr, _ = node3.stress([ 'read', 'n=1k', 'no-warmup', '-schema', 'replication(factor=2)', '-rate', 'threads=8' ]) if stdout is not None: assert "FAILURE" not in stdout @since('2.2') def test_bootstrap_with_reset_bootstrap_state(self): """Test bootstrap with resetting bootstrap progress""" cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.set_configuration_options( values={'stream_throughput_outbound_megabits_per_sec': 1}) cluster.populate(2).start() node1 = cluster.nodes['node1'] node1.stress(['write', 'n=100K', '-schema', 'replication(factor=2)']) node1.flush() # kill node1 in the middle of streaming to let it fail t = InterruptBootstrap(node1) t.start() # start bootstrapping node3 and wait for streaming node3 = new_node(cluster) try: node3.start() except NodeError: pass # node doesn't start as expected t.join() node1.start() # restart node3 bootstrap with resetting bootstrap progress node3.stop(signal_event=signal.SIGKILL) mark = node3.mark_log() node3.start(jvm_args=["-Dcassandra.reset_bootstrap_progress=true"]) # check if we reset bootstrap state node3.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark) # wait for node3 ready to query, 180s as the node needs to bootstrap node3.wait_for_binary_interface(from_mark=mark, timeout=180) # check if 2nd bootstrap succeeded assert_bootstrap_state(self, node3, 'COMPLETED') def test_manual_bootstrap(self): """ Test adding a new node and bootstrapping it manually. No auto_bootstrap. This test also verify that all data are OK after the addition of the new node. @jira_ticket CASSANDRA-9022 """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(2).start() (node1, node2) = cluster.nodelist() node1.stress([ 'write', 'n=1K', 'no-warmup', '-schema', 'replication(factor=2)', '-rate', 'threads=1', '-pop', 'dist=UNIFORM(1..1000)' ]) session = self.patient_exclusive_cql_connection(node2) stress_table = 'keyspace1.standard1' original_rows = list(session.execute("SELECT * FROM %s" % stress_table)) # Add a new node node3 = new_node(cluster, bootstrap=False) node3.start(wait_for_binary_proto=True) node3.repair() node1.cleanup() current_rows = list(session.execute("SELECT * FROM %s" % stress_table)) assert original_rows == current_rows def test_local_quorum_bootstrap(self): """ Test that CL local_quorum works while a node is bootstrapping. @jira_ticket CASSANDRA-8058 """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate([1, 1]) cluster.start() node1 = cluster.nodes['node1'] yaml_config = """ # Create the keyspace and table keyspace: keyspace1 keyspace_definition: | CREATE KEYSPACE keyspace1 WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': 1, 'dc2': 1}; table: users table_definition: CREATE TABLE users ( username text, first_name text, last_name text, email text, PRIMARY KEY(username) ) WITH compaction = {'class':'SizeTieredCompactionStrategy'}; insert: partitions: fixed(1) batchtype: UNLOGGED queries: read: cql: select * from users where username = ? fields: samerow """ with tempfile.NamedTemporaryFile(mode='w+') as stress_config: stress_config.write(yaml_config) stress_config.flush() node1.stress([ 'user', 'profile=' + stress_config.name, 'n=200K', 'no-warmup', 'ops(insert=1)', '-rate', 'threads=10' ]) node3 = new_node(cluster, data_center='dc2') node3.start(jvm_args=["-Dcassandra.write_survey=true"], no_wait=True) node3_seen = False for _ in range(30): # give node3 up to 30 seconds to start ntout = node1.nodetool('status').stdout if re.search(r'UJ\s+' + node3.ip_addr, ntout): node3_seen = True break time.sleep(1) assert node3_seen, "expected {} in status:\n{}".format( node3.ip_addr, ntout) out, err, _ = node1.stress([ 'user', 'profile=' + stress_config.name, 'ops(insert=1)', 'n=10k', 'no-warmup', 'cl=LOCAL_QUORUM', '-rate', 'threads=10', '-errors', 'retries=2' ]) ntout = node1.nodetool('status').stdout assert re.search(r'UJ\s+' + node3.ip_addr, ntout), ntout logger.debug(out) assert_stderr_clean(err) regex = re.compile("Operation.+error inserting key.+Exception") failure = regex.search(str(out)) assert failure is None, "Error during stress while bootstrapping" def test_shutdown_wiped_node_cannot_join(self): self._wiped_node_cannot_join_test(gently=True) def test_killed_wiped_node_cannot_join(self): self._wiped_node_cannot_join_test(gently=False) def _wiped_node_cannot_join_test(self, gently): """ @jira_ticket CASSANDRA-9765 Test that if we stop a node and wipe its data then the node cannot join when it is not a seed. Test both a nice shutdown or a forced shutdown, via the gently parameter. """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(3) cluster.start() stress_table = 'keyspace1.standard1' # write some data node1 = cluster.nodelist()[0] node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8']) session = self.patient_cql_connection(node1) original_rows = list( session.execute("SELECT * FROM {}".format(stress_table, ))) # Add a new node, bootstrap=True ensures that it is not a seed node4 = new_node(cluster, bootstrap=True) node4.start(wait_for_binary_proto=True) session = self.patient_cql_connection(node4) assert original_rows == list( session.execute("SELECT * FROM {}".format(stress_table, ))) # Stop the new node and wipe its data node4.stop(gently=gently) self._cleanup(node4) # Now start it, it should not be allowed to join. mark = node4.mark_log() node4.start(no_wait=True, wait_other_notice=False) node4.watch_log_for( "A node with address {} already exists, cancelling join".format( node4.address_for_current_version_slashy()), from_mark=mark) def test_decommissioned_wiped_node_can_join(self): """ @jira_ticket CASSANDRA-9765 Test that if we decommission a node and then wipe its data, it can join the cluster. """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(3) cluster.start() stress_table = 'keyspace1.standard1' # write some data node1 = cluster.nodelist()[0] node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8']) session = self.patient_cql_connection(node1) original_rows = list( session.execute("SELECT * FROM {}".format(stress_table, ))) # Add a new node, bootstrap=True ensures that it is not a seed node4 = new_node(cluster, bootstrap=True) node4.start(wait_for_binary_proto=True) session = self.patient_cql_connection(node4) assert original_rows == list( session.execute("SELECT * FROM {}".format(stress_table, ))) # Decommission the new node and wipe its data node4.decommission() node4.stop() self._cleanup(node4) # Now start it, it should be allowed to join mark = node4.mark_log() node4.start() node4.watch_log_for("JOINING:", from_mark=mark) def test_decommissioned_wiped_node_can_gossip_to_single_seed(self): """ @jira_ticket CASSANDRA-8072 @jira_ticket CASSANDRA-8422 Test that if we decommission a node, kill it and wipe its data, it can join a cluster with a single seed node. """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(1) cluster.start() node1 = cluster.nodelist()[0] # Add a new node, bootstrap=True ensures that it is not a seed node2 = new_node(cluster, bootstrap=True) node2.start(wait_for_binary_proto=True) session = self.patient_cql_connection(node1) if cluster.version() >= '2.2': # reduce system_distributed RF to 2 so we don't require forceful decommission session.execute( "ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};" ) session.execute( "ALTER KEYSPACE system_traces WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};" ) # Decommision the new node and kill it logger.debug("Decommissioning & stopping node2") node2.decommission() node2.stop(wait_other_notice=False) # Wipe its data for data_dir in node2.data_directories(): logger.debug("Deleting {}".format(data_dir)) shutil.rmtree(data_dir) commitlog_dir = os.path.join(node2.get_path(), 'commitlogs') logger.debug("Deleting {}".format(commitlog_dir)) shutil.rmtree(commitlog_dir) # Now start it, it should be allowed to join mark = node2.mark_log() logger.debug("Restarting wiped node2") node2.start(wait_other_notice=False) node2.watch_log_for("JOINING:", from_mark=mark) def test_failed_bootstrap_wiped_node_can_join(self): """ @jira_ticket CASSANDRA-9765 Test that if a node fails to bootstrap, it can join the cluster even if the data is wiped. """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(1) cluster.set_configuration_options( values={'stream_throughput_outbound_megabits_per_sec': 1}) cluster.start() stress_table = 'keyspace1.standard1' # write some data, enough for the bootstrap to fail later on node1 = cluster.nodelist()[0] node1.stress(['write', 'n=100K', 'no-warmup', '-rate', 'threads=8']) node1.flush() session = self.patient_cql_connection(node1) original_rows = list( session.execute("SELECT * FROM {}".format(stress_table, ))) # Add a new node, bootstrap=True ensures that it is not a seed node2 = new_node(cluster, bootstrap=True) # kill node2 in the middle of bootstrap t = KillOnBootstrap(node2) t.start() node2.start() t.join() assert not node2.is_running() # wipe any data for node2 self._cleanup(node2) # Now start it again, it should be allowed to join mark = node2.mark_log() node2.start() node2.watch_log_for("JOINING:", from_mark=mark) @since('3.0') @ported_to_in_jvm('4.1') def test_node_cannot_join_as_hibernating_node_without_replace_address( self): """ @jira_ticket CASSANDRA-14559 Test that a node cannot bootstrap without replace_address if a hibernating node exists with that address """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(2) # Setting seed node to first node to make sure replaced node is not in own seed list cluster.set_configuration_options({ 'seed_provider': [{ 'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider', 'parameters': [{ 'seeds': '127.0.0.1' }] }] }) cluster.start() node1 = cluster.nodelist()[0] node2 = cluster.nodelist()[1] replacement_address = node2.address() node2.stop() jvm_option = 'replace_address' logger.debug( "Starting replacement node {} with jvm_option '{}={}'".format( replacement_address, jvm_option, replacement_address)) replacement_node = Node('replacement', cluster=self.cluster, auto_bootstrap=True, thrift_interface=None, storage_interface=(replacement_address, 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042)) cluster.add(replacement_node, False) extra_jvm_args = [] extra_jvm_args.extend([ "-Dcassandra.{}={}".format(jvm_option, replacement_address), "-Dcassandra.ring_delay_ms=10000", "-Dcassandra.broadcast_interval_ms=10000" ]) wait_other_notice = False wait_for_binary_proto = False # Killing node earlier in bootstrap to prevent node making it to 'normal' status. t = KillOnReadyToBootstrap(replacement_node) t.start() replacement_node.start(jvm_args=extra_jvm_args, wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice) t.join() logger.debug("Asserting that original replacement node is not running") assert not replacement_node.is_running() # Assert node is actually in hibernate for test to be accurate. logger.debug( "Asserting that node is actually in hibernate status for test accuracy" ) assert 'hibernate' in node1.nodetool("gossipinfo").stdout extra_jvm_args = [] extra_jvm_args.extend([ "-Dcassandra.ring_delay_ms=10000", "-Dcassandra.broadcast_interval_ms=10000" ]) logger.debug( "Starting blind replacement node {}".format(replacement_address)) blind_replacement_node = Node('blind_replacement', cluster=self.cluster, auto_bootstrap=True, thrift_interface=None, storage_interface=(replacement_address, 7000), jmx_port='7400', remote_debug_port='0', initial_token=None, binary_interface=(replacement_address, 9042)) cluster.add(blind_replacement_node, False) wait_other_notice = False wait_for_binary_proto = False blind_replacement_node.start( wait_for_binary_proto=wait_for_binary_proto, wait_other_notice=wait_other_notice) # Asserting that the new node has correct log entry self.assert_log_had_msg( blind_replacement_node, "A node with the same IP in hibernate status was detected", timeout=60) # Waiting two seconds to give node a chance to stop in case above assertion is True. # When this happens cassandra may not shut down fast enough and the below assertion fails. time.sleep(15) # Asserting that then new node is not running. # This tests the actual expected state as opposed to just checking for the existance of the above error message. assert not blind_replacement_node.is_running() @since('2.1.1') def test_simultaneous_bootstrap(self): """ Attempt to bootstrap two nodes at once, to assert the second bootstrapped node fails, and does not interfere. Start a one node cluster and run a stress write workload. Start up a second node, and wait for the first node to detect it has joined the cluster. While the second node is bootstrapping, start a third node. This should fail. @jira_ticket CASSANDRA-7069 @jira_ticket CASSANDRA-9484 """ bootstrap_error = "Other bootstrapping/leaving/moving nodes detected," \ " cannot bootstrap while cassandra.consistent.rangemovement is true" cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(1) cluster.start() node1, = cluster.nodelist() node1.stress([ 'write', 'n=500K', 'no-warmup', '-schema', 'replication(factor=1)', '-rate', 'threads=10' ]) node2 = new_node(cluster) node2.start() for _ in range(30): # wait until node2 shows up ntout = node1.nodetool('status').stdout if re.search(r'UJ\s+' + node2.ip_addr, ntout): break time.sleep(0.1) node3 = new_node(cluster, remote_debug_port='2003') try: node3.start(wait_other_notice=False, verbose=False) except NodeError: pass # node doesn't start as expected time.sleep(.5) node2.watch_log_for("Starting listening for CQL clients") node3.watch_log_for(bootstrap_error) session = self.patient_exclusive_cql_connection(node2) # Repeat the select count(*) query, to help catch # bugs like 9484, where count(*) fails at higher # data loads. for _ in range(5): assert_one(session, "SELECT count(*) from keyspace1.standard1", [500000], cl=ConsistencyLevel.ONE) def test_cleanup(self): """ @jira_ticket CASSANDRA-11179 Make sure we remove processed files during cleanup """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.set_configuration_options(values={'concurrent_compactors': 4}) cluster.populate(1) cluster.start() node1, = cluster.nodelist() for x in range(0, 5): node1.stress([ 'write', 'n=100k', 'no-warmup', '-schema', 'compaction(strategy=SizeTieredCompactionStrategy,enabled=false)', 'replication(factor=1)', '-rate', 'threads=10' ]) node1.flush() node2 = new_node(cluster) node2.start(wait_for_binary_proto=True) event = threading.Event() failed = threading.Event() jobs = 1 thread = threading.Thread( target=self._monitor_datadir, args=(node1, event, len(node1.get_sstables("keyspace1", "standard1")), jobs, failed)) thread.setDaemon(True) thread.start() node1.nodetool("cleanup -j {} keyspace1 standard1".format(jobs)) event.set() thread.join() assert not failed.is_set() def _monitor_datadir(self, node, event, basecount, jobs, failed): while True: sstables = [ s for s in node.get_sstables("keyspace1", "standard1") if "tmplink" not in s ] if len(sstables) > basecount + jobs: logger.error("---") for sstable in sstables: logger.error(sstable) logger.error("Current count is {}, basecount was {}".format( len(sstables), basecount)) failed.set() return if event.is_set(): return time.sleep(.1) def _cleanup(self, node): commitlog_dir = os.path.join(node.get_path(), 'commitlogs') for data_dir in node.data_directories(): logger.debug("Deleting {}".format(data_dir)) shutil.rmtree(data_dir) shutil.rmtree(commitlog_dir) @since('2.2') @pytest.mark.ported_to_in_jvm # see org.apache.cassandra.distributed.test.BootstrapBinaryDisabledTest def test_bootstrap_binary_disabled(self): """ Test binary while bootstrapping and streaming fails. This test was ported to jvm-dtest org.apache.cassandra.distributed.test.BootstrapBinaryDisabledTest, as of this writing there are a few limitations with jvm-dtest which requries this test to stay, namely vnode support (ci also tests under different configs). Once jvm-dtest supports vnodes, this test can go away in favor of that class. @jira_ticket CASSANDRA-14526, CASSANDRA-14525, CASSANDRA-16127 """ config = { 'authenticator': 'org.apache.cassandra.auth.PasswordAuthenticator', 'authorizer': 'org.apache.cassandra.auth.CassandraAuthorizer', 'role_manager': 'org.apache.cassandra.auth.CassandraRoleManager', 'permissions_validity_in_ms': 0, 'roles_validity_in_ms': 0 } cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(1) node1 = cluster.nodes['node1'] # set up byteman node1.byteman_port = '8100' node1.import_config_files() cluster.start() # kill stream to node2 in the middle of streaming to let it fail if cluster.version() < '4.0': node1.byteman_submit([self.byteman_submit_path_pre_4_0]) else: node1.byteman_submit([self.byteman_submit_path_4_0]) node1.stress([ 'write', 'n=1K', 'no-warmup', 'cl=ONE', '-schema', 'replication(factor=3)', '-rate', 'threads=50', '-mode', 'native', 'cql3', 'user=cassandra', 'password=cassandra' ]) cluster.flush() # start bootstrapping node2 and wait for streaming node2 = new_node(cluster) node2.set_configuration_options(values=config) node2.byteman_port = '8101' # set for when we add node3 node2.import_config_files() node2.start(jvm_args=["-Dcassandra.ring_delay_ms=5000"]) self.assert_log_had_msg(node2, 'Some data streaming failed') try: node2.nodetool('join') pytest.fail('nodetool should have errored and failed to join ring') except ToolError as t: assert "Cannot join the ring until bootstrap completes" in t.stdout node2.nodetool('bootstrap resume') node2.wait_for_binary_interface() assert_bootstrap_state(self, node2, 'COMPLETED', user='******', password='******') # Test write survey behaviour node3 = new_node(cluster) node3.set_configuration_options(values=config) # kill stream to node3 in the middle of streaming to let it fail if cluster.version() < '4.0': node1.byteman_submit([self.byteman_submit_path_pre_4_0]) node2.byteman_submit([self.byteman_submit_path_pre_4_0]) else: node1.byteman_submit([self.byteman_submit_path_4_0]) node2.byteman_submit([self.byteman_submit_path_4_0]) node3.start(jvm_args=[ "-Dcassandra.write_survey=true", "-Dcassandra.ring_delay_ms=5000" ]) self.assert_log_had_msg(node3, 'Some data streaming failed') self.assert_log_had_msg( node3, "Not starting client transports in write_survey mode as it's bootstrapping or auth is enabled" ) try: node3.nodetool('join') pytest.fail('nodetool should have errored and failed to join ring') except ToolError as t: assert "Cannot join the ring until bootstrap completes" in t.stdout node3.nodetool('bootstrap resume') self.assert_log_had_msg( node3, "Not starting client transports in write_survey mode as it's bootstrapping or auth is enabled" ) # Should succeed in joining node3.nodetool('join') self.assert_log_had_msg( node3, "Leaving write survey mode and joining ring at operator request") assert_bootstrap_state(self, node3, 'COMPLETED', user='******', password='******') node3.wait_for_binary_interface() @since('4.1') def test_invalid_host_id(self): """ @jira_ticket CASSANDRA-14582 Test that node fails to bootstrap if host id is invalid """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(1) cluster.start() node2 = new_node(cluster) try: node2.start( jvm_args=["-Dcassandra.host_id_first_boot=invalid-host-id"], wait_other_notice=False, wait_for_binary_proto=True) pytest.fail( 'Node should fail to bootstrap because host id set was invalid' ) except NodeError: pass # node does not start as expected @since('4.1') def test_host_id_override(self): """ @jira_ticket CASSANDRA-14582 Test that node persists host id """ cluster = self.cluster cluster.set_environment_variable( 'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True') cluster.populate(1) cluster.start() host_id = "06fc931f-33b5-4e22-0001-000000000001" node1 = cluster.nodes['node1'] node2 = new_node(cluster) node2.start( wait_for_binary_proto=True, wait_other_notice=True, jvm_args=["-Dcassandra.host_id_first_boot={}".format(host_id)]) address2 = "'{}'".format(node2.address()) # 1. wait for host_id setup node2.watch_log_for(host_id) # 2. check host_id in local table session2 = self.patient_exclusive_cql_connection(node2) assert_one(session2, "SELECT host_id FROM system.local", [uuid.UUID(host_id)]) # 3. check host_id in other node's table session1 = self.patient_exclusive_cql_connection(node1) assert_one( session1, "SELECT host_id FROM system.peers_v2 WHERE peer = {}".format( address2), [uuid.UUID(host_id)]) # restart node and repeat node2.stop() node2.start(wait_for_binary_proto=True, wait_other_notice=True) # 1. wait for host_id setup node2.watch_log_for(host_id) # 2. check host_id in local table session2 = self.patient_exclusive_cql_connection(node2) assert_one(session2, "SELECT host_id FROM system.local", [uuid.UUID(host_id)]) # 3. check host_id in other node's table session1 = self.patient_exclusive_cql_connection(node1) assert_one( session1, "SELECT host_id FROM system.peers_v2 WHERE peer = {}".format( address2), [uuid.UUID(host_id)]) # restart node with another host_id and repeat node2.stop() node2.start( wait_for_binary_proto=True, wait_other_notice=True, jvm_args=[ "-Dcassandra.host_id_first_boot=setting-new-host-id-first-boot" ]) # 1. wait for host_id setup node2.watch_log_for(host_id) # 2. check host_id in local table session2 = self.patient_exclusive_cql_connection(node2) assert_one(session2, "SELECT host_id FROM system.local", [uuid.UUID(host_id)]) # 3. check host_id in other node's table session1 = self.patient_exclusive_cql_connection(node1) assert_one( session1, "SELECT host_id FROM system.peers_v2 WHERE peer = {}".format( address2), [uuid.UUID(host_id)])
def test_resumable_decommission(self): """ @jira_ticket CASSANDRA-12008 Test decommission operation is resumable """ self.fixture_dtest_setup.ignore_log_patterns = [ r'Streaming error occurred', r'Error while decommissioning node', r'Remote peer 127.0.0.2 failed stream session', r'Remote peer 127.0.0.2:7000 failed stream session' ] cluster = self.cluster cluster.set_configuration_options( values={'stream_throughput_outbound_megabits_per_sec': 1}) cluster.populate(3, install_byteman=True).start() node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node2) # reduce system_distributed RF to 2 so we don't require forceful decommission session.execute( "ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'2'};" ) create_ks(session, 'ks', 2) create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ALL) # Execute first rebuild, should fail with pytest.raises(ToolError): if cluster.version() >= '4.0': script = [mk_bman_path('4.0/decommission_failure_inject.btm')] else: script = [ mk_bman_path('pre4.0/decommission_failure_inject.btm') ] node2.byteman_submit(script) node2.nodetool('decommission') # Make sure previous ToolError is due to decommission node2.watch_log_for('Error while decommissioning node') # Decommission again mark = node2.mark_log() node2.nodetool('decommission') # Check decommision is done and we skipped transfereed ranges node2.watch_log_for('DECOMMISSIONED', from_mark=mark) node2.grep_log( "Skipping transferred range .* of keyspace ks, endpoint {}".format( node2.address_for_current_version_slashy()), filename='debug.log') # Check data is correctly forwarded to node1 and node3 cluster.remove(node2) node3.stop(gently=False) session = self.patient_exclusive_cql_connection(node1) session.execute('USE ks') for i in range(0, 10000): query_c1c2(session, i, ConsistencyLevel.ONE) node1.stop(gently=False) node3.start() session.shutdown() mark = node3.mark_log() node3.watch_log_for('Starting listening for CQL clients', from_mark=mark) session = self.patient_exclusive_cql_connection(node3) session.execute('USE ks') for i in range(0, 10000): query_c1c2(session, i, ConsistencyLevel.ONE)
def set_nodes(self): self.node1, self.node2, self.node3 = self.nodes # Make sure digest is not attempted against the transient node self.node3.byteman_submit([mk_bman_path('throw_on_digest.btm')])
def _test_restart_failed_replace(self, mode): self.fixture_dtest_setup.ignore_log_patterns = list( self.fixture_dtest_setup.ignore_log_patterns) + [ r'Error while waiting on bootstrap to complete' ] self._setup(n=3, enable_byteman=True) self._insert_data(n="1k") initial_data = self._fetch_initial_data() self._stop_node_to_replace() logger.debug("Submitting byteman script to make stream fail") btmmark = self.query_node.mark_log() if self.cluster.version() < '4.0': self.query_node.byteman_submit( [mk_bman_path('pre4.0/stream_failure.btm')]) self._do_replace(jvm_option='replace_address_first_boot', opts={'streaming_socket_timeout_in_ms': 1000}, wait_for_binary_proto=False, wait_other_notice=True) else: self.query_node.byteman_submit( [mk_bman_path('4.0/stream_failure.btm')]) self._do_replace(jvm_option='replace_address_first_boot', wait_for_binary_proto=False, wait_other_notice=True) # Make sure bootstrap did not complete successfully self.query_node.watch_log_for("Triggering network failure", from_mark=btmmark) self.query_node.watch_log_for("Stream failed", from_mark=btmmark) self.replacement_node.watch_log_for("Stream failed") self.replacement_node.watch_log_for( "Some data streaming failed.*IN_PROGRESS$") if mode == 'reset_resume_state': mark = self.replacement_node.mark_log() logger.debug( "Restarting replacement node with -Dcassandra.reset_bootstrap_progress=true" ) # restart replacement node with resetting bootstrap state (with 180s timeout) self.replacement_node.stop() self.replacement_node.start(jvm_args=[ "-Dcassandra.replace_address_first_boot={}".format( self.replaced_node.address()), "-Dcassandra.reset_bootstrap_progress=true" ], wait_for_binary_proto=180) # check if we reset bootstrap state self.replacement_node.watch_log_for( "Resetting bootstrap progress to start fresh", from_mark=mark) elif mode == 'resume': logger.debug("Resuming failed bootstrap") self.replacement_node.nodetool('bootstrap resume') # check if we skipped already retrieved ranges self.replacement_node.watch_log_for( "already available. Skipping streaming.") self.replacement_node.watch_log_for("Resume complete") elif mode == 'wipe': self.replacement_node.stop() logger.debug("Waiting other nodes to detect node stopped") node_log_str = self.replacement_node.address_for_current_version_slashy( ) self.query_node.watch_log_for( "FatClient {} has been silent for 30000ms, removing from gossip" .format(node_log_str), timeout=120) self.query_node.watch_log_for( "Node {} failed during replace.".format(node_log_str), timeout=120, filename='debug.log') logger.debug("Restarting node after wiping data") self._cleanup(self.replacement_node) self.replacement_node.start(jvm_args=[ "-Dcassandra.replace_address_first_boot={}".format( self.replaced_node.address()) ], wait_for_binary_proto=120) else: raise RuntimeError('invalid mode value {mode}'.format(mode=mode)) # check if bootstrap succeeded assert_bootstrap_state(self, self.replacement_node, 'COMPLETED') logger.debug("Bootstrap finished successfully, verifying data.") self._verify_data(initial_data)
def test_resumable_rebuild(self): """ @jira_ticket CASSANDRA-10810 Test rebuild operation is resumable """ self.fixture_dtest_setup.ignore_log_patterns = list( self.fixture_dtest_setup.ignore_log_patterns ) + [ r'Error while rebuilding node', r'Streaming error occurred on session with peer 127.0.0.3', r'Remote peer 127.0.0.3 failed stream session', r'Streaming error occurred on session with peer 127.0.0.3:7000', r'Remote peer /?127.0.0.3:7000 failed stream session', r'Stream receive task .* already finished', r'stream operation from /?127.0.0.1:.* failed' ] cluster = self.cluster cluster.set_configuration_options(values={ 'endpoint_snitch': 'org.apache.cassandra.locator.PropertyFileSnitch' }) # Create 2 nodes on dc1 node1 = cluster.create_node('node1', False, ('127.0.0.1', 9160), ('127.0.0.1', 7000), '7100', '2000', None, binary_interface=('127.0.0.1', 9042)) node2 = cluster.create_node('node2', False, ('127.0.0.2', 9160), ('127.0.0.2', 7000), '7200', '2001', None, binary_interface=('127.0.0.2', 9042)) cluster.add(node1, True, data_center='dc1') cluster.add(node2, True, data_center='dc1') node1.start(wait_for_binary_proto=True) node2.start(wait_for_binary_proto=True) # Insert data into node1 and node2 session = self.patient_exclusive_cql_connection(node1) create_ks(session, 'ks', {'dc1': 1}) create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ALL) key = list(range(10000, 20000)) session = self.patient_exclusive_cql_connection(node2) session.execute('USE ks') insert_c1c2(session, keys=key, consistency=ConsistencyLevel.ALL) session.shutdown() # Create a new node3 on dc2 node3 = cluster.create_node('node3', False, ('127.0.0.3', 9160), ('127.0.0.3', 7000), '7300', '2002', None, binary_interface=('127.0.0.3', 9042), byteman_port='8300') cluster.add(node3, False, data_center='dc2') node3.start(wait_other_notice=False, wait_for_binary_proto=True) # Wait for snitch to be refreshed time.sleep(5) # Alter necessary keyspace for rebuild operation session = self.patient_exclusive_cql_connection(node3) session.execute( "ALTER KEYSPACE ks WITH REPLICATION = {'class':'NetworkTopologyStrategy', 'dc1':1, 'dc2':1};" ) session.execute( "ALTER KEYSPACE system_auth WITH REPLICATION = {'class':'NetworkTopologyStrategy', 'dc1':1, 'dc2':1};" ) # Path to byteman script which makes the streaming to node2 throw an exception, making rebuild fail if cluster.version() < '4.0': script = [ mk_bman_path('pre4.0/inject_failure_streaming_to_node2.btm') ] else: script = [ mk_bman_path('4.0/inject_failure_streaming_to_node2.btm') ] node3.byteman_submit(script) # First rebuild must fail and data must be incomplete with pytest.raises(ToolError, message='Unexpected: SUCCEED'): logger.debug('Executing first rebuild -> '), node3.nodetool('rebuild dc1') logger.debug('Expected: FAILED') session.execute('USE ks') with pytest.raises(AssertionError, message='Unexpected: COMPLETE'): logger.debug('Checking data is complete -> '), for i in range(0, 20000): query_c1c2(session, i, ConsistencyLevel.LOCAL_ONE) logger.debug('Expected: INCOMPLETE') logger.debug('Executing second rebuild -> '), node3.nodetool('rebuild dc1') logger.debug('Expected: SUCCEED') # Check all streaming sessions completed, streamed ranges are skipped and verify streamed data node3.watch_log_for('All sessions completed') node3.watch_log_for('Skipping streaming those ranges.') logger.debug('Checking data is complete -> '), for i in range(0, 20000): query_c1c2(session, i, ConsistencyLevel.LOCAL_ONE) logger.debug('Expected: COMPLETE')