def test_normal_read_repair(self):
""" test the normal case """
node1, node2, node3 = self.cluster.nodelist()
assert isinstance(node1, Node)
assert isinstance(node2, Node)
assert isinstance(node3, Node)
session = self.get_cql_connection(node1, timeout=2)
session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)"))
node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")
# re-enable writes
node2.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')])
node2.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')])
coordinator = node2
# Stop reads on coordinator in order to make sure we do not go through
# the messaging service for the local reads
with StorageProxy(node2) as storage_proxy, stop_reads(coordinator):
assert storage_proxy.blocking_read_repair == 0
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 0
session = self.get_cql_connection(coordinator)
expected = [kcv(1, 0, 1), kcv(1, 1, 2)]
results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1"))
assert listify(results) == expected
assert storage_proxy.blocking_read_repair == 1
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 0
def test_failed_read_repair(self):
"""
If none of the disagreeing nodes ack the repair mutation, the read should fail
"""
node1, node2, node3 = self.cluster.nodelist()
assert isinstance(node1, Node)
assert isinstance(node2, Node)
assert isinstance(node3, Node)
session = self.get_cql_connection(node1, timeout=2)
session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)"))
node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
node2.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')])
with raises(WriteTimeout):
session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)"))
node2.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')])
session = self.get_cql_connection(node2)
with StorageProxy(node2) as storage_proxy:
assert storage_proxy.blocking_read_repair == 0
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 0
with raises(ReadTimeout):
session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1"))
assert storage_proxy.blocking_read_repair > 0
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write > 0
def test_speculative_write(self):
""" if one node doesn't respond to a read repair mutation, it should be sent to the remaining node """
node1, node2, node3 = self.cluster.nodelist()
assert isinstance(node1, Node)
assert isinstance(node2, Node)
assert isinstance(node3, Node)
session = self.get_cql_connection(node1, timeout=2)
session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)"))
node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")
# re-enable writes on node 3, leave them off on node2
node2.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')])
node1.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')])
with StorageProxy(node1) as storage_proxy:
assert storage_proxy.blocking_read_repair == 0
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 0
session = self.get_cql_connection(node1)
expected = [kcv(1, 0, 1), kcv(1, 1, 2)]
results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1"))
assert listify(results) == expected
assert storage_proxy.blocking_read_repair == 1
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 1
def test_counter_leader_with_partial_view(self):
"""
Test leader election with a starting node.
Testing that nodes do not elect as mutation leader a node with a partial view on the cluster.
Note that byteman rules can be syntax checked via the following command:
sh ./bin/bytemancheck.sh -cp ~/path_to/apache-cassandra-3.0.14-SNAPSHOT.jar ~/path_to/rule.btm
@jira_ticket CASSANDRA-13043
"""
cluster = self.cluster
cluster.populate(3, use_vnodes=True, install_byteman=True)
nodes = cluster.nodelist()
# Have node 1 and 3 cheat a bit during the leader election for a counter mutation; note that cheating
# takes place iff there is an actual chance for node 2 to be picked.
if cluster.version() < '4.0':
nodes[0].update_startup_byteman_script(
mk_bman_path('pre4.0/election_counter_leader_favor_node2.btm'))
nodes[2].update_startup_byteman_script(
mk_bman_path('pre4.0/election_counter_leader_favor_node2.btm'))
else:
nodes[0].update_startup_byteman_script(
mk_bman_path('4.0/election_counter_leader_favor_node2.btm'))
nodes[2].update_startup_byteman_script(
mk_bman_path('4.0/election_counter_leader_favor_node2.btm'))
cluster.start()
session = self.patient_cql_connection(nodes[0])
create_ks(session, 'ks', 3)
create_cf(session,
'cf',
validation="CounterColumnType",
columns={'c': 'counter'})
# Now stop the node and restart but first install a rule to slow down how fast node 2 will update the list
# nodes that are alive
nodes[1].stop(wait=True, wait_other_notice=False)
nodes[1].update_startup_byteman_script(
mk_bman_path('gossip_alive_callback_sleep.btm'))
nodes[1].start(no_wait=True, wait_other_notice=False)
# Until node 2 is fully alive try to force other nodes to pick him as mutation leader.
# If CASSANDRA-13043 is fixed, they will not. Otherwise they will do, but since we are slowing down how
# fast node 2 updates the list of nodes that are alive, it will just have a partial view on the cluster
# and thus will raise an 'UnavailableException' exception.
nb_attempts = 50000
for i in range(0, nb_attempts):
# Change the name of the counter for the sake of randomization
q = SimpleStatement(
query_string=
"UPDATE ks.cf SET c = c + 1 WHERE key = 'counter_%d'" % i,
consistency_level=ConsistencyLevel.QUORUM)
session.execute(q)
def test_bootstrap_waits_for_streaming_to_finish(self):
"""
Test that bootstrap completes and is marked as such after streaming finishes.
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
logger.debug("Create a cluster")
cluster.populate(1)
node1 = cluster.nodelist()[0]
logger.debug("Start node 1")
node1.start(wait_for_binary_proto=True)
logger.debug("Insert 10k rows")
node1.stress([
'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema',
'replication(factor=2)'
])
logger.debug("Bootstrap node 2 with delay")
node2 = new_node(cluster, byteman_port='4200')
node2.update_startup_byteman_script(
mk_bman_path('bootstrap_5s_sleep.btm'))
node2.start(wait_for_binary_proto=True)
assert_bootstrap_state(self, node2, 'COMPLETED')
assert node2.grep_log('Bootstrap completed', filename='debug.log')
def test_transient_full_merge_read(self):
""" When reading, transient replica should serve a missing read """
for node in self.nodes:
self.assert_has_no_sstables(node)
tm = lambda n: self.table_metrics(n)
self.insert_row(1, 1, 1)
# Stop writes to the other full node
self.node2.byteman_submit([mk_bman_path('stop_writes.btm')])
self.insert_row(1, 2, 2)
self.assert_local_rows(self.node1, [[1, 1, 1], [1, 2, 2]])
self.assert_local_rows(self.node2, [[1, 1, 1]])
self.assert_local_rows(self.node3, [[1, 1, 1], [1, 2, 2]])
self.assert_local_rows(self.node4, [[1, 2, 2]])
self.assert_local_rows(self.node5, [[1, 2, 2]])
# Stop reads from the node that will hold the second row
self.node1.stop()
# Whether we're reading from the full node or from the transient node, we should get consistent results
for node in [self.node2, self.node3, self.node4, self.node5]:
assert_all(self.exclusive_cql_connection(node),
"SELECT * FROM %s.%s" % (self.keyspace, self.table),
[[1, 1, 1], [1, 2, 2]],
cl=ConsistencyLevel.QUORUM)
def _batchlog_replay_compatibility_test(self, coordinator_idx,
current_nodes, previous_version,
previous_nodes, protocol_version):
session = self.prepare_mixed(coordinator_idx,
current_nodes,
previous_version,
previous_nodes,
protocol_version=protocol_version,
install_byteman=True)
coordinator = self.cluster.nodelist()[coordinator_idx]
coordinator.byteman_submit(
[mk_bman_path('fail_after_batchlog_write.btm')])
logger.debug(
"Injected byteman scripts to enable batchlog replay {}".format(
coordinator.name))
query = """
BEGIN BATCH
INSERT INTO users (id, firstname, lastname) VALUES (0, 'Jack', 'Sparrow')
INSERT INTO users (id, firstname, lastname) VALUES (1, 'Will', 'Turner')
APPLY BATCH
"""
session.execute(query)
# batchlog replay skips over all entries that are younger than
# 2 * write_request_timeout_in_ms ms: 1x timeout for all mutations to be written,
# and another 1x timeout for batch remove mutation to be received.
delay = 2 * coordinator.get_conf_option(
'write_request_timeout_in_ms') / 1000.0 + 1
logger.debug(
'Sleeping for {}s for the batches to not be skipped'.format(delay))
time.sleep(delay)
total_batches_replayed = 0
blm = make_mbean('db', type='BatchlogManager')
for n in self.cluster.nodelist():
if n == coordinator:
continue
with JolokiaAgent(n) as jmx:
logger.debug('Forcing batchlog replay for {}'.format(n.name))
jmx.execute_method(blm, 'forceBatchlogReplay')
batches_replayed = jmx.read_attribute(blm,
'TotalBatchesReplayed')
logger.debug('{} batches replayed on node {}'.format(
batches_replayed, n.name))
total_batches_replayed += batches_replayed
assert total_batches_replayed >= 2
for node in self.cluster.nodelist():
session = self.patient_exclusive_cql_connection(
node, protocol_version=protocol_version)
rows = sorted(
session.execute(
'SELECT id, firstname, lastname FROM ks.users'))
assert [[0, 'Jack', 'Sparrow'],
[1, 'Will', 'Turner']], [list(rows[0]) == list(rows[1])]
def test_speculative_write(self):
""" if a full replica isn't responding, we should send the write to the transient replica """
session = self.exclusive_cql_connection(self.node1)
self.node2.byteman_submit([mk_bman_path('slow_writes.btm')])
self.insert_row(1, 1, 1, session=session)
self.assert_local_rows(self.node1, [[1, 1, 1]])
self.assert_local_rows(self.node2, [])
self.assert_local_rows(self.node3, [[1, 1, 1]])
def fixture_set_cluster_settings(self, fixture_dtest_setup):
cluster = fixture_dtest_setup.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False,
'dynamic_snitch': False,
'write_request_timeout_in_ms': 1000,
'read_request_timeout_in_ms': 1000})
cluster.populate(3, install_byteman=True, debug=True)
byteman_validate(cluster.nodelist()[0], mk_bman_path('read_repair/sorted_live_endpoints.btm'), verbose=True)
cluster.start(jvm_args=['-XX:-PerfDisableSharedMem'])
session = fixture_dtest_setup.patient_exclusive_cql_connection(cluster.nodelist()[0], timeout=2)
session.execute("CREATE KEYSPACE ks WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k int, c int, v int, primary key (k, c)) WITH speculative_retry = '400ms';")
def test_simple_bootstrap_small_keepalive_period(self):
"""
@jira_ticket CASSANDRA-11841
Test that bootstrap completes if it takes longer than streaming_socket_timeout_in_ms or
2*streaming_keep_alive_period_in_secs to receive a single sstable
"""
cluster = self.cluster
yaml_opts = {'streaming_keep_alive_period_in_secs': 2}
if cluster.version() < '4.0':
yaml_opts['streaming_socket_timeout_in_ms'] = 1000
cluster.set_configuration_options(values=yaml_opts)
# Create a single node cluster
cluster.populate(1)
node1 = cluster.nodelist()[0]
logger.debug("Setting up byteman on {}".format(node1.name))
# set up byteman
node1.byteman_port = '8100'
node1.import_config_files()
cluster.start()
# Create more than one sstable larger than 1MB
node1.stress([
'write', 'n=1K', '-rate', 'threads=8', '-schema',
'compaction(strategy=SizeTieredCompactionStrategy, enabled=false)'
])
cluster.flush()
logger.debug("Submitting byteman script to {} to".format(node1.name))
# Sleep longer than streaming_socket_timeout_in_ms to make sure the node will not be killed
node1.byteman_submit([mk_bman_path('stream_5s_sleep.btm')])
# Bootstraping a new node with very small streaming_socket_timeout_in_ms
node2 = new_node(cluster)
node2.start(wait_for_binary_proto=True)
# Shouldn't fail due to streaming socket timeout timeout
assert_bootstrap_state(self, node2, 'COMPLETED')
if cluster.version() < '4.0':
for node in cluster.nodelist():
assert node.grep_log(
'Scheduling keep-alive task with 2s period.',
filename='debug.log')
assert node.grep_log('Sending keep-alive',
filename='debug.log')
assert node.grep_log('Received keep-alive',
filename='debug.log')
def _test_speculative_write_repair_cycle(self,
primary_range,
optimized_repair,
repair_coordinator,
expect_node3_data,
use_lcs=False):
"""
if one of the full replicas is not available, data should be written to the transient replica, but removed after incremental repair
"""
for node in self.nodes:
self.assert_has_no_sstables(node)
if use_lcs:
self.use_lcs()
self.node2.byteman_submit([mk_bman_path('stop_writes.btm')])
# self.insert_row(1)
tm = lambda n: self.table_metrics(n)
with tm(self.node1) as tm1, tm(self.node2) as tm2, tm(
self.node3) as tm3:
assert tm1.write_count == 0
assert tm2.write_count == 0
assert tm3.write_count == 0
self.insert_row(1, 1, 1)
assert tm1.write_count == 1
assert tm2.write_count == 0
assert tm3.write_count == 1
self.assert_has_sstables(self.node1, flush=True)
self.assert_has_no_sstables(self.node2, flush=True)
self.assert_has_sstables(self.node3, flush=True)
repair_opts = ['repair', self.keyspace]
if primary_range: repair_opts.append('-pr')
if optimized_repair: repair_opts.append('-os')
self.node1.nodetool(' '.join(repair_opts))
self.assert_has_sstables(self.node1, compact=True)
self.assert_has_sstables(self.node2, compact=True)
if expect_node3_data:
self.assert_has_sstables(self.node3, compact=True)
else:
self.assert_has_no_sstables(self.node3, compact=True)
entire_sstable = "true" if self.stream_entire_sstables() else "false"
assert self.node2.grep_log(
'Incoming stream entireSSTable={}'.format(entire_sstable),
filename='debug.log')
def test_custom_speculate(self):
""" If write can't succeed on full replica, it's written to the transient node instead """
session = self.exclusive_cql_connection(self.node1)
session.execute(
"ALTER TABLE %s.%s WITH speculative_retry = '99.99PERCENTILE';" %
(self.keyspace, self.table))
self.insert_row(1, 1, 1)
# Stop writes to the other full node
self.node2.byteman_submit([mk_bman_path('stop_writes.btm')])
self.insert_row(1, 2, 2)
for node in self.nodes:
assert_all(self.exclusive_cql_connection(node),
"SELECT * FROM %s.%s WHERE pk = 1" %
(self.keyspace, self.table), [[1, 1, 1], [1, 2, 2]],
cl=ConsistencyLevel.QUORUM)
def test_quorum_requirement_on_speculated_read(self):
"""
Even if we speculate on every stage, we should still only require a quorum of responses for success
"""
node1, node2, node3 = self.cluster.nodelist()
assert isinstance(node1, Node)
assert isinstance(node2, Node)
assert isinstance(node3, Node)
session = self.get_cql_connection(node1, timeout=2)
session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)"))
node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")
# re-enable writes
node2.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')])
node3.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')])
# force endpoint order
node1.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')])
node2.byteman_submit([mk_bman_path('read_repair/stop_digest_reads.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_data_reads.btm')])
node2.byteman_submit([mk_bman_path('read_repair/stop_rr_writes.btm')])
with StorageProxy(node1) as storage_proxy:
assert storage_proxy.get_table_metric("ks", "tbl", "SpeculativeRetries") == 0
assert storage_proxy.blocking_read_repair == 0
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 0
session = self.get_cql_connection(node1)
expected = [kcv(1, 0, 1), kcv(1, 1, 2)]
results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1"))
assert listify(results) == expected
assert storage_proxy.get_table_metric("ks", "tbl", "SpeculativeRetries") == 1
assert storage_proxy.blocking_read_repair == 1
assert storage_proxy.speculated_rr_read == 0 # there shouldn't be any replicas to speculate on
assert storage_proxy.speculated_rr_write == 1
def _test_missed_update_with_transient_replicas(self, missed_by_transient):
cluster = self.cluster
cluster.set_configuration_options(
values={
'hinted_handoff_enabled':
False,
'num_tokens':
1,
'commitlog_sync_period_in_ms':
500,
'enable_transient_replication':
True,
'partitioner':
'org.apache.cassandra.dht.OrderPreservingPartitioner'
})
cluster.set_batch_commitlog(enabled=True)
cluster.populate(2, tokens=[0, 1], debug=True, install_byteman=True)
node1, node2 = cluster.nodelist()
cluster.start()
self.session = self.patient_exclusive_cql_connection(
node1, consistency_level=CL.ALL)
self.session.execute(
"CREATE KEYSPACE %s WITH replication = "
"{'class': 'SimpleStrategy', 'replication_factor': '2/1'}" %
(keyspace))
self.session.execute("USE " + keyspace)
self.session.execute("CREATE TABLE t (k int PRIMARY KEY, v text)"
" WITH speculative_retry = 'NEVER'"
" AND additional_write_policy = 'NEVER'"
" AND read_repair = 'NONE'")
# insert in both nodes with CL=ALL
self.session.execute("INSERT INTO t(k, v) VALUES (0, 'old')")
# update the previous value with CL=ONE only in one replica
node = cluster.nodelist()[1 if missed_by_transient else 0]
node.byteman_submit([mk_bman_path('stop_writes.btm')])
self.session.execute(
SimpleStatement("UPDATE t SET v = 'new' WHERE k = 0",
consistency_level=CL.ONE))
# query with CL=ALL to verify that no old values are resurrected
self._assert_none("SELECT * FROM t WHERE v = 'old'")
self._assert_one("SELECT * FROM t WHERE v = 'new'", row=[0, 'new'])
def test_speculative_data_request(self):
""" If one node doesn't respond to a full data request, it should query the other """
node1, node2, node3 = self.cluster.nodelist()
assert isinstance(node1, Node)
assert isinstance(node2, Node)
assert isinstance(node3, Node)
session = self.get_cql_connection(node1, timeout=2)
session.execute(quorum("INSERT INTO ks.tbl (k, c, v) VALUES (1, 0, 1)"))
node2.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
node3.byteman_submit([mk_bman_path('read_repair/stop_writes.btm')])
session.execute("INSERT INTO ks.tbl (k, c, v) VALUES (1, 1, 2)")
# re-enable writes
node2.byteman_submit(['-u', mk_bman_path('read_repair/stop_writes.btm')])
node1.byteman_submit([mk_bman_path('read_repair/sorted_live_endpoints.btm')])
version = self.cluster.cassandra_version()
if version < '4.1':
node1.byteman_submit([mk_bman_path('request_verb_timing.btm')])
else:
node1.byteman_submit([mk_bman_path('post4.0/request_verb_timing.btm')])
with StorageProxy(node1) as storage_proxy:
assert storage_proxy.blocking_read_repair == 0
assert storage_proxy.speculated_rr_read == 0
assert storage_proxy.speculated_rr_write == 0
session = self.get_cql_connection(node1)
node2.byteman_submit([mk_bman_path('read_repair/stop_data_reads.btm')])
results = session.execute(quorum("SELECT * FROM ks.tbl WHERE k=1"))
timing = request_verb_timing(node1)
repair_req_node3 = timing[node3.ip_addr].get('READ_REPAIR_REQ')
repair_req_node2 = timing[node2.ip_addr].get('READ_REPAIR_REQ')
assert listify(results) == [kcv(1, 0, 1), kcv(1, 1, 2)]
assert storage_proxy.blocking_read_repair == 1
assert storage_proxy.speculated_rr_read == 1
# under normal circumstances we don't expect a speculated write here,
# but the repair request to node 3 may timeout due to CPU contention and
# then a speculated write is sent to node 2, so we just make sure that the
# request to node 2 didn't happen before the request to node 3
assert storage_proxy.speculated_rr_write == 0 or repair_req_node2 > repair_req_node3
def _byteman_cycle(nodes, scripts):
script_path = lambda name: mk_bman_path('read_repair/') + name + '.btm'
for script in scripts:
byteman_validate(nodes[0], script_path(script))
for node in nodes:
assert isinstance(node, Node)
for name in scripts:
print(node.name)
node.byteman_submit([script_path(name)])
yield
for node in nodes:
for name in scripts:
print(node.name)
node.byteman_submit(['-u', script_path(name)])
def _test_transient_full_merge_read_with_delete(self, coordinator):
""" When reading, transient replica should serve a missing read """
for node in self.nodes:
self.assert_has_no_sstables(node)
tm = lambda n: self.table_metrics(n)
self.insert_row(1, 1, 1)
self.insert_row(1, 2, 2)
# Stop writes to the other full node
self.node2.byteman_submit([mk_bman_path('stop_writes.btm')])
self.delete_row(1, 2)
self.assert_local_rows(self.node3, [])
# Stop reads from the node that will hold the second row
self.node1.stop()
assert_all(self.exclusive_cql_connection(coordinator),
"SELECT * FROM %s.%s" % (self.keyspace, self.table),
[[1, 1, 1]],
cl=ConsistencyLevel.QUORUM)
def test_transient_write(self):
""" If write can't succeed on full replica, it's written to the transient node instead """
for node in self.nodes:
self.assert_has_no_sstables(node)
tm = lambda n: self.table_metrics(n)
with tm(self.node1) as tm1, tm(self.node2) as tm2, tm(
self.node3) as tm3:
self.insert_row(1, 1, 1)
# Stop writes to the other full node
self.node2.byteman_submit([mk_bman_path('stop_writes.btm')])
self.insert_row(1, 2, 2)
# node1 should contain both rows
self.assert_local_rows(self.node1, [[1, 1, 1], [1, 2, 2]])
# write couldn't succeed on node2, so it has only the first row
self.assert_local_rows(self.node2, [[1, 1, 1]])
# transient replica should hold only the second row
self.assert_local_rows(self.node3, [[1, 2, 2]])
def test_sstableloader_with_failing_2i(self):
"""
@jira_ticket CASSANDRA-10130
Simulates an index building failure during SSTables load.
The table data should be loaded and the index should be marked for rebuilding during the next node start.
"""
def create_schema_with_2i(session):
create_ks(session, 'k', 1)
session.execute(
"CREATE TABLE k.t (p int, c int, v int, PRIMARY KEY(p, c))")
session.execute("CREATE INDEX idx ON k.t(v)")
cluster = self.cluster
cluster.populate(1, install_byteman=True).start()
node = cluster.nodelist()[0]
session = self.patient_cql_connection(node)
create_schema_with_2i(session)
session.execute("INSERT INTO k.t(p, c, v) VALUES (0, 1, 8)")
# Stop node and copy SSTables
node.nodetool('drain')
node.stop()
self.copy_sstables(cluster, node)
# Wipe out data and restart
cluster.clear()
cluster.start()
# Restore the schema
session = self.patient_cql_connection(node)
create_schema_with_2i(session)
# The table should exist and be empty, and the index should be empty and marked as built
assert_one(
session,
"""SELECT * FROM system."IndexInfo" WHERE table_name='k'""",
['k', 'idx', None])
assert_none(session, "SELECT * FROM k.t")
assert_none(session, "SELECT * FROM k.t WHERE v = 8")
# Add some additional data before loading the SSTable, to check that it will be still accessible
session.execute("INSERT INTO k.t(p, c, v) VALUES (0, 2, 8)")
assert_one(session, "SELECT * FROM k.t", [0, 2, 8])
assert_one(session, "SELECT * FROM k.t WHERE v = 8", [0, 2, 8])
# Load SSTables with a failure during index creation
node.byteman_submit([mk_bman_path('index_build_failure.btm')])
with pytest.raises(Exception):
self.load_sstables(cluster, node, 'k')
# Check that the index isn't marked as built and the old SSTable data has been loaded but not indexed
assert_none(
session,
"""SELECT * FROM system."IndexInfo" WHERE table_name='k'""")
assert_all(session, "SELECT * FROM k.t", [[0, 1, 8], [0, 2, 8]])
assert_one(session, "SELECT * FROM k.t WHERE v = 8", [0, 2, 8])
# Restart the node to trigger index rebuild
node.nodetool('drain')
node.stop()
cluster.start()
session = self.patient_cql_connection(node)
# Check that the index is marked as built and the index has been rebuilt
assert_one(
session,
"""SELECT * FROM system."IndexInfo" WHERE table_name='k'""",
['k', 'idx', None])
assert_all(session, "SELECT * FROM k.t", [[0, 1, 8], [0, 2, 8]])
assert_all(session, "SELECT * FROM k.t WHERE v = 8",
[[0, 1, 8], [0, 2, 8]])
class BootstrapTester(Tester):
byteman_submit_path_pre_4_0 = mk_bman_path('pre4.0/stream_failure.btm')
byteman_submit_path_4_0 = mk_bman_path('4.0/stream_failure.btm')
@pytest.fixture(autouse=True)
def fixture_add_additional_log_patterns(self, fixture_dtest_setup):
fixture_dtest_setup.allow_log_errors = True
fixture_dtest_setup.ignore_log_patterns = (
# This one occurs when trying to send the migration to a
# node that hasn't started yet, and when it does, it gets
# replayed and everything is fine.
r'Can\'t send migration request: node.*is down',
# ignore streaming error during bootstrap
r'Exception encountered during startup',
r'Streaming error occurred')
def _base_bootstrap_test(self,
bootstrap=None,
bootstrap_from_version=None,
enable_ssl=None):
def default_bootstrap(cluster, token):
node2 = new_node(cluster)
node2.set_configuration_options(values={'initial_token': token})
node2.start(wait_for_binary_proto=True)
return node2
if bootstrap is None:
bootstrap = default_bootstrap
cluster = self.cluster
if enable_ssl:
logger.debug("***using internode ssl***")
generate_ssl_stores(self.fixture_dtest_setup.test_path)
cluster.enable_internode_ssl(self.fixture_dtest_setup.test_path)
tokens = cluster.balanced_tokens(2)
cluster.set_configuration_options(values={'num_tokens': 1})
logger.debug("[node1, node2] tokens: %r" % (tokens, ))
keys = 10000
# Create a single node cluster
cluster.populate(1)
node1 = cluster.nodelist()[0]
if bootstrap_from_version:
logger.debug("starting source node on version {}".format(
bootstrap_from_version))
node1.set_install_dir(version=bootstrap_from_version)
node1.set_configuration_options(values={'initial_token': tokens[0]})
cluster.start()
session = self.patient_cql_connection(node1)
create_ks(session, 'ks', 1)
create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
# record the size before inserting any of our own data
empty_size = data_size(node1, 'ks', 'cf')
logger.debug("node1 empty size for ks.cf: %s" % float(empty_size))
insert_statement = session.prepare(
"INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')")
execute_concurrent_with_args(session, insert_statement,
[['k%d' % k] for k in range(keys)])
node1.flush()
node1.compact()
initial_size = data_size(node1, 'ks', 'cf')
logger.debug("node1 size for ks.cf before bootstrapping node2: %s" %
float(initial_size))
# Reads inserted data all during the bootstrap process. We shouldn't
# get any error
query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE)
session.shutdown()
# Bootstrapping a new node in the current version
node2 = bootstrap(cluster, tokens[1])
node2.compact()
node1.cleanup()
logger.debug("node1 size for ks.cf after cleanup: %s" %
float(data_size(node1, 'ks', 'cf')))
node1.compact()
logger.debug("node1 size for ks.cf after compacting: %s" %
float(data_size(node1, 'ks', 'cf')))
logger.debug("node2 size for ks.cf after compacting: %s" %
float(data_size(node2, 'ks', 'cf')))
size1 = float(data_size(node1, 'ks', 'cf'))
size2 = float(data_size(node2, 'ks', 'cf'))
assert_almost_equal(size1, size2, error=0.3)
assert_almost_equal(float(initial_size - empty_size),
2 * (size1 - float(empty_size)))
assert_bootstrap_state(self, node2, 'COMPLETED')
@pytest.mark.no_vnodes
def test_simple_bootstrap_with_ssl(self):
self._base_bootstrap_test(enable_ssl=True)
@pytest.mark.no_vnodes
def test_simple_bootstrap(self):
self._base_bootstrap_test()
@pytest.mark.no_vnodes
def test_bootstrap_on_write_survey(self):
def bootstrap_on_write_survey_and_join(cluster, token):
node2 = new_node(cluster)
node2.set_configuration_options(values={'initial_token': token})
node2.start(jvm_args=["-Dcassandra.write_survey=true"],
wait_for_binary_proto=True)
assert len(
node2.grep_log(
'Startup complete, but write survey mode is active, not becoming an active ring member.'
))
assert_bootstrap_state(self, node2, 'IN_PROGRESS')
node2.nodetool("join")
assert len(
node2.grep_log(
'Leaving write survey mode and joining ring at operator request'
))
return node2
self._base_bootstrap_test(bootstrap_on_write_survey_and_join)
def _test_bootstrap_with_compatibility_flag_on(self,
bootstrap_from_version):
def bootstrap_with_compatibility_flag_on(cluster, token):
node2 = new_node(cluster)
node2.set_configuration_options(values={'initial_token': token})
# cassandra.force_3_0_protocol_version parameter is needed to allow schema
# changes during the bootstrapping for upgrades from 3.0.14+ to anything upwards for 3.0.x or 3.x clusters.
# @jira_ticket CASSANDRA-13004 for detailed context on `cassandra.force_3_0_protocol_version` flag
node2.start(
jvm_args=["-Dcassandra.force_3_0_protocol_version=true"],
wait_for_binary_proto=True)
return node2
self._base_bootstrap_test(
bootstrap_with_compatibility_flag_on,
bootstrap_from_version=bootstrap_from_version)
@since('3.10')
@pytest.mark.no_vnodes
def test_simple_bootstrap_small_keepalive_period(self):
"""
@jira_ticket CASSANDRA-11841
Test that bootstrap completes if it takes longer than streaming_socket_timeout_in_ms or
2*streaming_keep_alive_period_in_secs to receive a single sstable
"""
cluster = self.cluster
yaml_opts = {'streaming_keep_alive_period_in_secs': 2}
if cluster.version() < '4.0':
yaml_opts['streaming_socket_timeout_in_ms'] = 1000
cluster.set_configuration_options(values=yaml_opts)
# Create a single node cluster
cluster.populate(1)
node1 = cluster.nodelist()[0]
logger.debug("Setting up byteman on {}".format(node1.name))
# set up byteman
node1.byteman_port = '8100'
node1.import_config_files()
cluster.start()
# Create more than one sstable larger than 1MB
node1.stress([
'write', 'n=1K', '-rate', 'threads=8', '-schema',
'compaction(strategy=SizeTieredCompactionStrategy, enabled=false)'
])
cluster.flush()
logger.debug("Submitting byteman script to {} to".format(node1.name))
# Sleep longer than streaming_socket_timeout_in_ms to make sure the node will not be killed
node1.byteman_submit([mk_bman_path('stream_5s_sleep.btm')])
# Bootstraping a new node with very small streaming_socket_timeout_in_ms
node2 = new_node(cluster)
node2.start(wait_for_binary_proto=True)
# Shouldn't fail due to streaming socket timeout timeout
assert_bootstrap_state(self, node2, 'COMPLETED')
if cluster.version() < '4.0':
for node in cluster.nodelist():
assert node.grep_log(
'Scheduling keep-alive task with 2s period.',
filename='debug.log')
assert node.grep_log('Sending keep-alive',
filename='debug.log')
assert node.grep_log('Received keep-alive',
filename='debug.log')
def test_simple_bootstrap_nodata(self):
"""
@jira_ticket CASSANDRA-11010
Test that bootstrap completes if streaming from nodes with no data
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
# Create a two-node cluster
cluster.populate(2)
cluster.start()
# Bootstrapping a new node
node3 = new_node(cluster)
node3.start(wait_for_binary_proto=True)
assert_bootstrap_state(self, node3, 'COMPLETED')
def test_schema_removed_nodes(self):
"""
@jira_ticket CASSANDRA-16577
Test that nodes can bootstrap after a schema change performed with a node removed
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2)
cluster.start()
node1, node2 = cluster.nodelist()
node2.decommission(force=cluster.version() > '4')
session = self.patient_cql_connection(node1)
session.execute(
"CREATE KEYSPACE k WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};"
)
node3 = new_node(cluster)
node3.start(wait_for_binary_proto=True)
def test_read_from_bootstrapped_node(self):
"""
Test bootstrapped node sees existing data
@jira_ticket CASSANDRA-6648
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(3)
cluster.start()
node1 = cluster.nodes['node1']
node1.stress([
'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema',
'replication(factor=2)'
])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
original_rows = list(
session.execute("SELECT * FROM %s" % (stress_table, )))
node4 = new_node(cluster)
node4.start(wait_for_binary_proto=True)
session = self.patient_exclusive_cql_connection(node4)
new_rows = list(session.execute("SELECT * FROM %s" % (stress_table, )))
assert original_rows == new_rows
@since('3.0')
def test_bootstrap_waits_for_streaming_to_finish(self):
"""
Test that bootstrap completes and is marked as such after streaming finishes.
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
logger.debug("Create a cluster")
cluster.populate(1)
node1 = cluster.nodelist()[0]
logger.debug("Start node 1")
node1.start(wait_for_binary_proto=True)
logger.debug("Insert 10k rows")
node1.stress([
'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema',
'replication(factor=2)'
])
logger.debug("Bootstrap node 2 with delay")
node2 = new_node(cluster, byteman_port='4200')
node2.update_startup_byteman_script(
mk_bman_path('bootstrap_5s_sleep.btm'))
node2.start(wait_for_binary_proto=True)
assert_bootstrap_state(self, node2, 'COMPLETED')
assert node2.grep_log('Bootstrap completed', filename='debug.log')
def test_consistent_range_movement_true_with_replica_down_should_fail(
self):
self._bootstrap_test_with_replica_down(True)
def test_consistent_range_movement_false_with_replica_down_should_succeed(
self):
self._bootstrap_test_with_replica_down(False)
def test_consistent_range_movement_true_with_rf1_should_fail(self):
self._bootstrap_test_with_replica_down(True, rf=1)
def test_consistent_range_movement_false_with_rf1_should_succeed(self):
self._bootstrap_test_with_replica_down(False, rf=1)
def test_rf_gt_nodes_multidc_should_succeed(self):
"""
Validating a KS with RF > N on multi DC doesn't break bootstrap
@jira_ticket CASSANDRA-16296 CASSANDRA-16411
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate([1, 1])
cluster.start()
node1 = cluster.nodelist()[0]
node2 = cluster.nodelist()[1]
session = self.patient_exclusive_cql_connection(node1)
session.execute(
"CREATE KEYSPACE k WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'dc1' : '3'}"
)
if cluster.version() >= '4.0':
warning = 'Your replication factor 3 for keyspace k is higher than the number of nodes 1 for datacenter dc1'
assert len(node1.grep_log(warning)) == 1
assert len(node2.grep_log(warning)) == 0
session.execute(
"ALTER KEYSPACE k WITH REPLICATION = {'class' : 'NetworkTopologyStrategy', 'dc1' : '2'}"
)
session.execute(
"CREATE TABLE k.testgtrfmultidc (KEY text PRIMARY KEY)")
session.execute(
"INSERT INTO k.testgtrfmultidc (KEY) VALUES ('test_rf_gt_nodes_multidc_should_succeed')"
)
if cluster.version() >= '4.0':
warning = 'Your replication factor 2 for keyspace k is higher than the number of nodes 1 for datacenter dc1'
assert len(node1.grep_log(warning)) == 1
assert len(node2.grep_log(warning)) == 0
marks = map(lambda n: n.mark_log(), cluster.nodelist())
node3 = Node(name='node3',
cluster=cluster,
auto_bootstrap=True,
thrift_interface=('127.0.0.3', 9160),
storage_interface=('127.0.0.3', 7000),
jmx_port='7300',
remote_debug_port='0',
initial_token=None,
binary_interface=('127.0.0.3', 9042))
cluster.add(node3, is_seed=False, data_center="dc1")
node3.start(wait_for_binary_proto=True)
if cluster.version() >= '4.0':
warning = 'is higher than the number of nodes'
for (node, mark) in zip(cluster.nodelist(), marks):
assert len(node.grep_log(warning, from_mark=mark)) == 0
session3 = self.patient_exclusive_cql_connection(node3)
assert_one(session3, "SELECT * FROM k.testgtrfmultidc",
["test_rf_gt_nodes_multidc_should_succeed"])
def _bootstrap_test_with_replica_down(self,
consistent_range_movement,
rf=2):
"""
Test to check consistent bootstrap will not succeed when there are insufficient replicas
@jira_ticket CASSANDRA-11848
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2)
node1, node2 = cluster.nodelist()
node3_token = None
# Make token assignment deterministic
if not self.dtest_config.use_vnodes:
cluster.set_configuration_options(values={'num_tokens': 1})
tokens = cluster.balanced_tokens(3)
logger.debug("non-vnode tokens: %r" % (tokens, ))
node1.set_configuration_options(
values={'initial_token': tokens[0]})
node2.set_configuration_options(
values={'initial_token': tokens[2]})
node3_token = tokens[1] # Add node 3 between node1 and node2
cluster.start()
node1.stress([
'write', 'n=10K', 'no-warmup', '-rate', 'threads=8', '-schema',
'replication(factor={})'.format(rf)
])
# change system_auth keyspace to 2 (default is 1) to avoid
# "Unable to find sufficient sources for streaming" warning
if cluster.cassandra_version() >= '2.2.0':
session = self.patient_cql_connection(node1)
session.execute("""
ALTER KEYSPACE system_auth
WITH replication = {'class':'SimpleStrategy', 'replication_factor':2};
""")
# Stop node2, so node3 will not be able to perform consistent range movement
node2.stop(wait_other_notice=True)
successful_bootstrap_expected = not consistent_range_movement
node3 = new_node(cluster, token=node3_token)
node3.start(wait_for_binary_proto=successful_bootstrap_expected,
wait_other_notice=successful_bootstrap_expected,
jvm_args=[
"-Dcassandra.consistent.rangemovement={}".format(
consistent_range_movement)
])
if successful_bootstrap_expected:
# with rf=1 and cassandra.consistent.rangemovement=false, missing sources are ignored
if not consistent_range_movement and rf == 1:
node3.watch_log_for(
"Unable to find sufficient sources for streaming range")
assert node3.is_running()
assert_bootstrap_state(self, node3, 'COMPLETED')
else:
if consistent_range_movement:
if cluster.version() < '4.0':
node3.watch_log_for(
"A node required to move the data consistently is down"
)
else:
node3.watch_log_for(
"Necessary replicas for strict consistency were removed by source filters"
)
else:
node3.watch_log_for(
"Unable to find sufficient sources for streaming range")
assert_not_running(node3)
@since('2.2')
def test_resumable_bootstrap(self):
"""
Test resuming bootstrap after data streaming failure
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2)
node1 = cluster.nodes['node1']
# set up byteman
node1.byteman_port = '8100'
node1.import_config_files()
cluster.start()
# kill stream to node3 in the middle of streaming to let it fail
if cluster.version() < '4.0':
node1.byteman_submit([self.byteman_submit_path_pre_4_0])
else:
node1.byteman_submit([self.byteman_submit_path_4_0])
node1.stress([
'write', 'n=1K', 'no-warmup', 'cl=TWO', '-schema',
'replication(factor=2)', '-rate', 'threads=50'
])
cluster.flush()
# start bootstrapping node3 and wait for streaming
node3 = new_node(cluster)
node3.start(wait_other_notice=False)
# let streaming fail as we expect
node3.watch_log_for('Some data streaming failed')
# bring back node3 and invoke nodetool bootstrap to resume bootstrapping
node3.nodetool('bootstrap resume')
node3.wait_for_binary_interface()
assert_bootstrap_state(self, node3, 'COMPLETED')
# cleanup to guarantee each node will only have sstables of its ranges
cluster.cleanup()
logger.debug("Check data is present")
# Let's check stream bootstrap completely transferred data
stdout, stderr, _ = node3.stress([
'read', 'n=1k', 'no-warmup', '-schema', 'replication(factor=2)',
'-rate', 'threads=8'
])
if stdout is not None:
assert "FAILURE" not in stdout
@since('2.2')
def test_bootstrap_with_reset_bootstrap_state(self):
"""Test bootstrap with resetting bootstrap progress"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.set_configuration_options(
values={'stream_throughput_outbound_megabits_per_sec': 1})
cluster.populate(2).start()
node1 = cluster.nodes['node1']
node1.stress(['write', 'n=100K', '-schema', 'replication(factor=2)'])
node1.flush()
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# start bootstrapping node3 and wait for streaming
node3 = new_node(cluster)
try:
node3.start()
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node3 bootstrap with resetting bootstrap progress
node3.stop(signal_event=signal.SIGKILL)
mark = node3.mark_log()
node3.start(jvm_args=["-Dcassandra.reset_bootstrap_progress=true"])
# check if we reset bootstrap state
node3.watch_log_for("Resetting bootstrap progress to start fresh",
from_mark=mark)
# wait for node3 ready to query, 180s as the node needs to bootstrap
node3.wait_for_binary_interface(from_mark=mark, timeout=180)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node3, 'COMPLETED')
def test_manual_bootstrap(self):
"""
Test adding a new node and bootstrapping it manually. No auto_bootstrap.
This test also verify that all data are OK after the addition of the new node.
@jira_ticket CASSANDRA-9022
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2).start()
(node1, node2) = cluster.nodelist()
node1.stress([
'write', 'n=1K', 'no-warmup', '-schema', 'replication(factor=2)',
'-rate', 'threads=1', '-pop', 'dist=UNIFORM(1..1000)'
])
session = self.patient_exclusive_cql_connection(node2)
stress_table = 'keyspace1.standard1'
original_rows = list(session.execute("SELECT * FROM %s" %
stress_table))
# Add a new node
node3 = new_node(cluster, bootstrap=False)
node3.start(wait_for_binary_proto=True)
node3.repair()
node1.cleanup()
current_rows = list(session.execute("SELECT * FROM %s" % stress_table))
assert original_rows == current_rows
def test_local_quorum_bootstrap(self):
"""
Test that CL local_quorum works while a node is bootstrapping.
@jira_ticket CASSANDRA-8058
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate([1, 1])
cluster.start()
node1 = cluster.nodes['node1']
yaml_config = """
# Create the keyspace and table
keyspace: keyspace1
keyspace_definition: |
CREATE KEYSPACE keyspace1 WITH replication = {'class': 'NetworkTopologyStrategy', 'dc1': 1, 'dc2': 1};
table: users
table_definition:
CREATE TABLE users (
username text,
first_name text,
last_name text,
email text,
PRIMARY KEY(username)
) WITH compaction = {'class':'SizeTieredCompactionStrategy'};
insert:
partitions: fixed(1)
batchtype: UNLOGGED
queries:
read:
cql: select * from users where username = ?
fields: samerow
"""
with tempfile.NamedTemporaryFile(mode='w+') as stress_config:
stress_config.write(yaml_config)
stress_config.flush()
node1.stress([
'user', 'profile=' + stress_config.name, 'n=200K', 'no-warmup',
'ops(insert=1)', '-rate', 'threads=10'
])
node3 = new_node(cluster, data_center='dc2')
node3.start(jvm_args=["-Dcassandra.write_survey=true"],
no_wait=True)
node3_seen = False
for _ in range(30): # give node3 up to 30 seconds to start
ntout = node1.nodetool('status').stdout
if re.search(r'UJ\s+' + node3.ip_addr, ntout):
node3_seen = True
break
time.sleep(1)
assert node3_seen, "expected {} in status:\n{}".format(
node3.ip_addr, ntout)
out, err, _ = node1.stress([
'user', 'profile=' + stress_config.name, 'ops(insert=1)',
'n=10k', 'no-warmup', 'cl=LOCAL_QUORUM', '-rate', 'threads=10',
'-errors', 'retries=2'
])
ntout = node1.nodetool('status').stdout
assert re.search(r'UJ\s+' + node3.ip_addr, ntout), ntout
logger.debug(out)
assert_stderr_clean(err)
regex = re.compile("Operation.+error inserting key.+Exception")
failure = regex.search(str(out))
assert failure is None, "Error during stress while bootstrapping"
def test_shutdown_wiped_node_cannot_join(self):
self._wiped_node_cannot_join_test(gently=True)
def test_killed_wiped_node_cannot_join(self):
self._wiped_node_cannot_join_test(gently=False)
def _wiped_node_cannot_join_test(self, gently):
"""
@jira_ticket CASSANDRA-9765
Test that if we stop a node and wipe its data then the node cannot join
when it is not a seed. Test both a nice shutdown or a forced shutdown, via
the gently parameter.
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(3)
cluster.start()
stress_table = 'keyspace1.standard1'
# write some data
node1 = cluster.nodelist()[0]
node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8'])
session = self.patient_cql_connection(node1)
original_rows = list(
session.execute("SELECT * FROM {}".format(stress_table, )))
# Add a new node, bootstrap=True ensures that it is not a seed
node4 = new_node(cluster, bootstrap=True)
node4.start(wait_for_binary_proto=True)
session = self.patient_cql_connection(node4)
assert original_rows == list(
session.execute("SELECT * FROM {}".format(stress_table, )))
# Stop the new node and wipe its data
node4.stop(gently=gently)
self._cleanup(node4)
# Now start it, it should not be allowed to join.
mark = node4.mark_log()
node4.start(no_wait=True, wait_other_notice=False)
node4.watch_log_for(
"A node with address {} already exists, cancelling join".format(
node4.address_for_current_version_slashy()),
from_mark=mark)
def test_decommissioned_wiped_node_can_join(self):
"""
@jira_ticket CASSANDRA-9765
Test that if we decommission a node and then wipe its data, it can join the cluster.
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(3)
cluster.start()
stress_table = 'keyspace1.standard1'
# write some data
node1 = cluster.nodelist()[0]
node1.stress(['write', 'n=10K', 'no-warmup', '-rate', 'threads=8'])
session = self.patient_cql_connection(node1)
original_rows = list(
session.execute("SELECT * FROM {}".format(stress_table, )))
# Add a new node, bootstrap=True ensures that it is not a seed
node4 = new_node(cluster, bootstrap=True)
node4.start(wait_for_binary_proto=True)
session = self.patient_cql_connection(node4)
assert original_rows == list(
session.execute("SELECT * FROM {}".format(stress_table, )))
# Decommission the new node and wipe its data
node4.decommission()
node4.stop()
self._cleanup(node4)
# Now start it, it should be allowed to join
mark = node4.mark_log()
node4.start()
node4.watch_log_for("JOINING:", from_mark=mark)
def test_decommissioned_wiped_node_can_gossip_to_single_seed(self):
"""
@jira_ticket CASSANDRA-8072
@jira_ticket CASSANDRA-8422
Test that if we decommission a node, kill it and wipe its data, it can join a cluster with a single
seed node.
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(1)
cluster.start()
node1 = cluster.nodelist()[0]
# Add a new node, bootstrap=True ensures that it is not a seed
node2 = new_node(cluster, bootstrap=True)
node2.start(wait_for_binary_proto=True)
session = self.patient_cql_connection(node1)
if cluster.version() >= '2.2':
# reduce system_distributed RF to 2 so we don't require forceful decommission
session.execute(
"ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};"
)
session.execute(
"ALTER KEYSPACE system_traces WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'1'};"
)
# Decommision the new node and kill it
logger.debug("Decommissioning & stopping node2")
node2.decommission()
node2.stop(wait_other_notice=False)
# Wipe its data
for data_dir in node2.data_directories():
logger.debug("Deleting {}".format(data_dir))
shutil.rmtree(data_dir)
commitlog_dir = os.path.join(node2.get_path(), 'commitlogs')
logger.debug("Deleting {}".format(commitlog_dir))
shutil.rmtree(commitlog_dir)
# Now start it, it should be allowed to join
mark = node2.mark_log()
logger.debug("Restarting wiped node2")
node2.start(wait_other_notice=False)
node2.watch_log_for("JOINING:", from_mark=mark)
def test_failed_bootstrap_wiped_node_can_join(self):
"""
@jira_ticket CASSANDRA-9765
Test that if a node fails to bootstrap, it can join the cluster even if the data is wiped.
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(1)
cluster.set_configuration_options(
values={'stream_throughput_outbound_megabits_per_sec': 1})
cluster.start()
stress_table = 'keyspace1.standard1'
# write some data, enough for the bootstrap to fail later on
node1 = cluster.nodelist()[0]
node1.stress(['write', 'n=100K', 'no-warmup', '-rate', 'threads=8'])
node1.flush()
session = self.patient_cql_connection(node1)
original_rows = list(
session.execute("SELECT * FROM {}".format(stress_table, )))
# Add a new node, bootstrap=True ensures that it is not a seed
node2 = new_node(cluster, bootstrap=True)
# kill node2 in the middle of bootstrap
t = KillOnBootstrap(node2)
t.start()
node2.start()
t.join()
assert not node2.is_running()
# wipe any data for node2
self._cleanup(node2)
# Now start it again, it should be allowed to join
mark = node2.mark_log()
node2.start()
node2.watch_log_for("JOINING:", from_mark=mark)
@since('3.0')
@ported_to_in_jvm('4.1')
def test_node_cannot_join_as_hibernating_node_without_replace_address(
self):
"""
@jira_ticket CASSANDRA-14559
Test that a node cannot bootstrap without replace_address if a hibernating node exists with that address
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(2)
# Setting seed node to first node to make sure replaced node is not in own seed list
cluster.set_configuration_options({
'seed_provider': [{
'class_name':
'org.apache.cassandra.locator.SimpleSeedProvider',
'parameters': [{
'seeds': '127.0.0.1'
}]
}]
})
cluster.start()
node1 = cluster.nodelist()[0]
node2 = cluster.nodelist()[1]
replacement_address = node2.address()
node2.stop()
jvm_option = 'replace_address'
logger.debug(
"Starting replacement node {} with jvm_option '{}={}'".format(
replacement_address, jvm_option, replacement_address))
replacement_node = Node('replacement',
cluster=self.cluster,
auto_bootstrap=True,
thrift_interface=None,
storage_interface=(replacement_address, 7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=(replacement_address, 9042))
cluster.add(replacement_node, False)
extra_jvm_args = []
extra_jvm_args.extend([
"-Dcassandra.{}={}".format(jvm_option, replacement_address),
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"
])
wait_other_notice = False
wait_for_binary_proto = False
# Killing node earlier in bootstrap to prevent node making it to 'normal' status.
t = KillOnReadyToBootstrap(replacement_node)
t.start()
replacement_node.start(jvm_args=extra_jvm_args,
wait_for_binary_proto=wait_for_binary_proto,
wait_other_notice=wait_other_notice)
t.join()
logger.debug("Asserting that original replacement node is not running")
assert not replacement_node.is_running()
# Assert node is actually in hibernate for test to be accurate.
logger.debug(
"Asserting that node is actually in hibernate status for test accuracy"
)
assert 'hibernate' in node1.nodetool("gossipinfo").stdout
extra_jvm_args = []
extra_jvm_args.extend([
"-Dcassandra.ring_delay_ms=10000",
"-Dcassandra.broadcast_interval_ms=10000"
])
logger.debug(
"Starting blind replacement node {}".format(replacement_address))
blind_replacement_node = Node('blind_replacement',
cluster=self.cluster,
auto_bootstrap=True,
thrift_interface=None,
storage_interface=(replacement_address,
7000),
jmx_port='7400',
remote_debug_port='0',
initial_token=None,
binary_interface=(replacement_address,
9042))
cluster.add(blind_replacement_node, False)
wait_other_notice = False
wait_for_binary_proto = False
blind_replacement_node.start(
wait_for_binary_proto=wait_for_binary_proto,
wait_other_notice=wait_other_notice)
# Asserting that the new node has correct log entry
self.assert_log_had_msg(
blind_replacement_node,
"A node with the same IP in hibernate status was detected",
timeout=60)
# Waiting two seconds to give node a chance to stop in case above assertion is True.
# When this happens cassandra may not shut down fast enough and the below assertion fails.
time.sleep(15)
# Asserting that then new node is not running.
# This tests the actual expected state as opposed to just checking for the existance of the above error message.
assert not blind_replacement_node.is_running()
@since('2.1.1')
def test_simultaneous_bootstrap(self):
"""
Attempt to bootstrap two nodes at once, to assert the second bootstrapped node fails, and does not interfere.
Start a one node cluster and run a stress write workload.
Start up a second node, and wait for the first node to detect it has joined the cluster.
While the second node is bootstrapping, start a third node. This should fail.
@jira_ticket CASSANDRA-7069
@jira_ticket CASSANDRA-9484
"""
bootstrap_error = "Other bootstrapping/leaving/moving nodes detected," \
" cannot bootstrap while cassandra.consistent.rangemovement is true"
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(1)
cluster.start()
node1, = cluster.nodelist()
node1.stress([
'write', 'n=500K', 'no-warmup', '-schema', 'replication(factor=1)',
'-rate', 'threads=10'
])
node2 = new_node(cluster)
node2.start()
for _ in range(30): # wait until node2 shows up
ntout = node1.nodetool('status').stdout
if re.search(r'UJ\s+' + node2.ip_addr, ntout):
break
time.sleep(0.1)
node3 = new_node(cluster, remote_debug_port='2003')
try:
node3.start(wait_other_notice=False, verbose=False)
except NodeError:
pass # node doesn't start as expected
time.sleep(.5)
node2.watch_log_for("Starting listening for CQL clients")
node3.watch_log_for(bootstrap_error)
session = self.patient_exclusive_cql_connection(node2)
# Repeat the select count(*) query, to help catch
# bugs like 9484, where count(*) fails at higher
# data loads.
for _ in range(5):
assert_one(session,
"SELECT count(*) from keyspace1.standard1", [500000],
cl=ConsistencyLevel.ONE)
def test_cleanup(self):
"""
@jira_ticket CASSANDRA-11179
Make sure we remove processed files during cleanup
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.set_configuration_options(values={'concurrent_compactors': 4})
cluster.populate(1)
cluster.start()
node1, = cluster.nodelist()
for x in range(0, 5):
node1.stress([
'write', 'n=100k', 'no-warmup', '-schema',
'compaction(strategy=SizeTieredCompactionStrategy,enabled=false)',
'replication(factor=1)', '-rate', 'threads=10'
])
node1.flush()
node2 = new_node(cluster)
node2.start(wait_for_binary_proto=True)
event = threading.Event()
failed = threading.Event()
jobs = 1
thread = threading.Thread(
target=self._monitor_datadir,
args=(node1, event,
len(node1.get_sstables("keyspace1",
"standard1")), jobs, failed))
thread.setDaemon(True)
thread.start()
node1.nodetool("cleanup -j {} keyspace1 standard1".format(jobs))
event.set()
thread.join()
assert not failed.is_set()
def _monitor_datadir(self, node, event, basecount, jobs, failed):
while True:
sstables = [
s for s in node.get_sstables("keyspace1", "standard1")
if "tmplink" not in s
]
if len(sstables) > basecount + jobs:
logger.error("---")
for sstable in sstables:
logger.error(sstable)
logger.error("Current count is {}, basecount was {}".format(
len(sstables), basecount))
failed.set()
return
if event.is_set():
return
time.sleep(.1)
def _cleanup(self, node):
commitlog_dir = os.path.join(node.get_path(), 'commitlogs')
for data_dir in node.data_directories():
logger.debug("Deleting {}".format(data_dir))
shutil.rmtree(data_dir)
shutil.rmtree(commitlog_dir)
@since('2.2')
@pytest.mark.ported_to_in_jvm # see org.apache.cassandra.distributed.test.BootstrapBinaryDisabledTest
def test_bootstrap_binary_disabled(self):
"""
Test binary while bootstrapping and streaming fails.
This test was ported to jvm-dtest org.apache.cassandra.distributed.test.BootstrapBinaryDisabledTest,
as of this writing there are a few limitations with jvm-dtest which requries this test to
stay, namely vnode support (ci also tests under different configs). Once jvm-dtest supports
vnodes, this test can go away in favor of that class.
@jira_ticket CASSANDRA-14526, CASSANDRA-14525, CASSANDRA-16127
"""
config = {
'authenticator': 'org.apache.cassandra.auth.PasswordAuthenticator',
'authorizer': 'org.apache.cassandra.auth.CassandraAuthorizer',
'role_manager': 'org.apache.cassandra.auth.CassandraRoleManager',
'permissions_validity_in_ms': 0,
'roles_validity_in_ms': 0
}
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(1)
node1 = cluster.nodes['node1']
# set up byteman
node1.byteman_port = '8100'
node1.import_config_files()
cluster.start()
# kill stream to node2 in the middle of streaming to let it fail
if cluster.version() < '4.0':
node1.byteman_submit([self.byteman_submit_path_pre_4_0])
else:
node1.byteman_submit([self.byteman_submit_path_4_0])
node1.stress([
'write', 'n=1K', 'no-warmup', 'cl=ONE', '-schema',
'replication(factor=3)', '-rate', 'threads=50', '-mode', 'native',
'cql3', 'user=cassandra', 'password=cassandra'
])
cluster.flush()
# start bootstrapping node2 and wait for streaming
node2 = new_node(cluster)
node2.set_configuration_options(values=config)
node2.byteman_port = '8101' # set for when we add node3
node2.import_config_files()
node2.start(jvm_args=["-Dcassandra.ring_delay_ms=5000"])
self.assert_log_had_msg(node2, 'Some data streaming failed')
try:
node2.nodetool('join')
pytest.fail('nodetool should have errored and failed to join ring')
except ToolError as t:
assert "Cannot join the ring until bootstrap completes" in t.stdout
node2.nodetool('bootstrap resume')
node2.wait_for_binary_interface()
assert_bootstrap_state(self,
node2,
'COMPLETED',
user='******',
password='******')
# Test write survey behaviour
node3 = new_node(cluster)
node3.set_configuration_options(values=config)
# kill stream to node3 in the middle of streaming to let it fail
if cluster.version() < '4.0':
node1.byteman_submit([self.byteman_submit_path_pre_4_0])
node2.byteman_submit([self.byteman_submit_path_pre_4_0])
else:
node1.byteman_submit([self.byteman_submit_path_4_0])
node2.byteman_submit([self.byteman_submit_path_4_0])
node3.start(jvm_args=[
"-Dcassandra.write_survey=true", "-Dcassandra.ring_delay_ms=5000"
])
self.assert_log_had_msg(node3, 'Some data streaming failed')
self.assert_log_had_msg(
node3,
"Not starting client transports in write_survey mode as it's bootstrapping or auth is enabled"
)
try:
node3.nodetool('join')
pytest.fail('nodetool should have errored and failed to join ring')
except ToolError as t:
assert "Cannot join the ring until bootstrap completes" in t.stdout
node3.nodetool('bootstrap resume')
self.assert_log_had_msg(
node3,
"Not starting client transports in write_survey mode as it's bootstrapping or auth is enabled"
)
# Should succeed in joining
node3.nodetool('join')
self.assert_log_had_msg(
node3,
"Leaving write survey mode and joining ring at operator request")
assert_bootstrap_state(self,
node3,
'COMPLETED',
user='******',
password='******')
node3.wait_for_binary_interface()
@since('4.1')
def test_invalid_host_id(self):
"""
@jira_ticket CASSANDRA-14582
Test that node fails to bootstrap if host id is invalid
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(1)
cluster.start()
node2 = new_node(cluster)
try:
node2.start(
jvm_args=["-Dcassandra.host_id_first_boot=invalid-host-id"],
wait_other_notice=False,
wait_for_binary_proto=True)
pytest.fail(
'Node should fail to bootstrap because host id set was invalid'
)
except NodeError:
pass # node does not start as expected
@since('4.1')
def test_host_id_override(self):
"""
@jira_ticket CASSANDRA-14582
Test that node persists host id
"""
cluster = self.cluster
cluster.set_environment_variable(
'CASSANDRA_TOKEN_PREGENERATION_DISABLED', 'True')
cluster.populate(1)
cluster.start()
host_id = "06fc931f-33b5-4e22-0001-000000000001"
node1 = cluster.nodes['node1']
node2 = new_node(cluster)
node2.start(
wait_for_binary_proto=True,
wait_other_notice=True,
jvm_args=["-Dcassandra.host_id_first_boot={}".format(host_id)])
address2 = "'{}'".format(node2.address())
# 1. wait for host_id setup
node2.watch_log_for(host_id)
# 2. check host_id in local table
session2 = self.patient_exclusive_cql_connection(node2)
assert_one(session2, "SELECT host_id FROM system.local",
[uuid.UUID(host_id)])
# 3. check host_id in other node's table
session1 = self.patient_exclusive_cql_connection(node1)
assert_one(
session1,
"SELECT host_id FROM system.peers_v2 WHERE peer = {}".format(
address2), [uuid.UUID(host_id)])
# restart node and repeat
node2.stop()
node2.start(wait_for_binary_proto=True, wait_other_notice=True)
# 1. wait for host_id setup
node2.watch_log_for(host_id)
# 2. check host_id in local table
session2 = self.patient_exclusive_cql_connection(node2)
assert_one(session2, "SELECT host_id FROM system.local",
[uuid.UUID(host_id)])
# 3. check host_id in other node's table
session1 = self.patient_exclusive_cql_connection(node1)
assert_one(
session1,
"SELECT host_id FROM system.peers_v2 WHERE peer = {}".format(
address2), [uuid.UUID(host_id)])
# restart node with another host_id and repeat
node2.stop()
node2.start(
wait_for_binary_proto=True,
wait_other_notice=True,
jvm_args=[
"-Dcassandra.host_id_first_boot=setting-new-host-id-first-boot"
])
# 1. wait for host_id setup
node2.watch_log_for(host_id)
# 2. check host_id in local table
session2 = self.patient_exclusive_cql_connection(node2)
assert_one(session2, "SELECT host_id FROM system.local",
[uuid.UUID(host_id)])
# 3. check host_id in other node's table
session1 = self.patient_exclusive_cql_connection(node1)
assert_one(
session1,
"SELECT host_id FROM system.peers_v2 WHERE peer = {}".format(
address2), [uuid.UUID(host_id)])
def test_resumable_decommission(self):
"""
@jira_ticket CASSANDRA-12008
Test decommission operation is resumable
"""
self.fixture_dtest_setup.ignore_log_patterns = [
r'Streaming error occurred', r'Error while decommissioning node',
r'Remote peer 127.0.0.2 failed stream session',
r'Remote peer 127.0.0.2:7000 failed stream session'
]
cluster = self.cluster
cluster.set_configuration_options(
values={'stream_throughput_outbound_megabits_per_sec': 1})
cluster.populate(3, install_byteman=True).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_cql_connection(node2)
# reduce system_distributed RF to 2 so we don't require forceful decommission
session.execute(
"ALTER KEYSPACE system_distributed WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor':'2'};"
)
create_ks(session, 'ks', 2)
create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ALL)
# Execute first rebuild, should fail
with pytest.raises(ToolError):
if cluster.version() >= '4.0':
script = [mk_bman_path('4.0/decommission_failure_inject.btm')]
else:
script = [
mk_bman_path('pre4.0/decommission_failure_inject.btm')
]
node2.byteman_submit(script)
node2.nodetool('decommission')
# Make sure previous ToolError is due to decommission
node2.watch_log_for('Error while decommissioning node')
# Decommission again
mark = node2.mark_log()
node2.nodetool('decommission')
# Check decommision is done and we skipped transfereed ranges
node2.watch_log_for('DECOMMISSIONED', from_mark=mark)
node2.grep_log(
"Skipping transferred range .* of keyspace ks, endpoint {}".format(
node2.address_for_current_version_slashy()),
filename='debug.log')
# Check data is correctly forwarded to node1 and node3
cluster.remove(node2)
node3.stop(gently=False)
session = self.patient_exclusive_cql_connection(node1)
session.execute('USE ks')
for i in range(0, 10000):
query_c1c2(session, i, ConsistencyLevel.ONE)
node1.stop(gently=False)
node3.start()
session.shutdown()
mark = node3.mark_log()
node3.watch_log_for('Starting listening for CQL clients',
from_mark=mark)
session = self.patient_exclusive_cql_connection(node3)
session.execute('USE ks')
for i in range(0, 10000):
query_c1c2(session, i, ConsistencyLevel.ONE)
def set_nodes(self):
self.node1, self.node2, self.node3 = self.nodes
# Make sure digest is not attempted against the transient node
self.node3.byteman_submit([mk_bman_path('throw_on_digest.btm')])
def _test_restart_failed_replace(self, mode):
self.fixture_dtest_setup.ignore_log_patterns = list(
self.fixture_dtest_setup.ignore_log_patterns) + [
r'Error while waiting on bootstrap to complete'
]
self._setup(n=3, enable_byteman=True)
self._insert_data(n="1k")
initial_data = self._fetch_initial_data()
self._stop_node_to_replace()
logger.debug("Submitting byteman script to make stream fail")
btmmark = self.query_node.mark_log()
if self.cluster.version() < '4.0':
self.query_node.byteman_submit(
[mk_bman_path('pre4.0/stream_failure.btm')])
self._do_replace(jvm_option='replace_address_first_boot',
opts={'streaming_socket_timeout_in_ms': 1000},
wait_for_binary_proto=False,
wait_other_notice=True)
else:
self.query_node.byteman_submit(
[mk_bman_path('4.0/stream_failure.btm')])
self._do_replace(jvm_option='replace_address_first_boot',
wait_for_binary_proto=False,
wait_other_notice=True)
# Make sure bootstrap did not complete successfully
self.query_node.watch_log_for("Triggering network failure",
from_mark=btmmark)
self.query_node.watch_log_for("Stream failed", from_mark=btmmark)
self.replacement_node.watch_log_for("Stream failed")
self.replacement_node.watch_log_for(
"Some data streaming failed.*IN_PROGRESS$")
if mode == 'reset_resume_state':
mark = self.replacement_node.mark_log()
logger.debug(
"Restarting replacement node with -Dcassandra.reset_bootstrap_progress=true"
)
# restart replacement node with resetting bootstrap state (with 180s timeout)
self.replacement_node.stop()
self.replacement_node.start(jvm_args=[
"-Dcassandra.replace_address_first_boot={}".format(
self.replaced_node.address()),
"-Dcassandra.reset_bootstrap_progress=true"
],
wait_for_binary_proto=180)
# check if we reset bootstrap state
self.replacement_node.watch_log_for(
"Resetting bootstrap progress to start fresh", from_mark=mark)
elif mode == 'resume':
logger.debug("Resuming failed bootstrap")
self.replacement_node.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
self.replacement_node.watch_log_for(
"already available. Skipping streaming.")
self.replacement_node.watch_log_for("Resume complete")
elif mode == 'wipe':
self.replacement_node.stop()
logger.debug("Waiting other nodes to detect node stopped")
node_log_str = self.replacement_node.address_for_current_version_slashy(
)
self.query_node.watch_log_for(
"FatClient {} has been silent for 30000ms, removing from gossip"
.format(node_log_str),
timeout=120)
self.query_node.watch_log_for(
"Node {} failed during replace.".format(node_log_str),
timeout=120,
filename='debug.log')
logger.debug("Restarting node after wiping data")
self._cleanup(self.replacement_node)
self.replacement_node.start(jvm_args=[
"-Dcassandra.replace_address_first_boot={}".format(
self.replaced_node.address())
],
wait_for_binary_proto=120)
else:
raise RuntimeError('invalid mode value {mode}'.format(mode=mode))
# check if bootstrap succeeded
assert_bootstrap_state(self, self.replacement_node, 'COMPLETED')
logger.debug("Bootstrap finished successfully, verifying data.")
self._verify_data(initial_data)
def test_resumable_rebuild(self):
"""
@jira_ticket CASSANDRA-10810
Test rebuild operation is resumable
"""
self.fixture_dtest_setup.ignore_log_patterns = list(
self.fixture_dtest_setup.ignore_log_patterns
) + [
r'Error while rebuilding node',
r'Streaming error occurred on session with peer 127.0.0.3',
r'Remote peer 127.0.0.3 failed stream session',
r'Streaming error occurred on session with peer 127.0.0.3:7000',
r'Remote peer /?127.0.0.3:7000 failed stream session',
r'Stream receive task .* already finished',
r'stream operation from /?127.0.0.1:.* failed'
]
cluster = self.cluster
cluster.set_configuration_options(values={
'endpoint_snitch':
'org.apache.cassandra.locator.PropertyFileSnitch'
})
# Create 2 nodes on dc1
node1 = cluster.create_node('node1',
False, ('127.0.0.1', 9160),
('127.0.0.1', 7000),
'7100',
'2000',
None,
binary_interface=('127.0.0.1', 9042))
node2 = cluster.create_node('node2',
False, ('127.0.0.2', 9160),
('127.0.0.2', 7000),
'7200',
'2001',
None,
binary_interface=('127.0.0.2', 9042))
cluster.add(node1, True, data_center='dc1')
cluster.add(node2, True, data_center='dc1')
node1.start(wait_for_binary_proto=True)
node2.start(wait_for_binary_proto=True)
# Insert data into node1 and node2
session = self.patient_exclusive_cql_connection(node1)
create_ks(session, 'ks', {'dc1': 1})
create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'})
insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ALL)
key = list(range(10000, 20000))
session = self.patient_exclusive_cql_connection(node2)
session.execute('USE ks')
insert_c1c2(session, keys=key, consistency=ConsistencyLevel.ALL)
session.shutdown()
# Create a new node3 on dc2
node3 = cluster.create_node('node3',
False, ('127.0.0.3', 9160),
('127.0.0.3', 7000),
'7300',
'2002',
None,
binary_interface=('127.0.0.3', 9042),
byteman_port='8300')
cluster.add(node3, False, data_center='dc2')
node3.start(wait_other_notice=False, wait_for_binary_proto=True)
# Wait for snitch to be refreshed
time.sleep(5)
# Alter necessary keyspace for rebuild operation
session = self.patient_exclusive_cql_connection(node3)
session.execute(
"ALTER KEYSPACE ks WITH REPLICATION = {'class':'NetworkTopologyStrategy', 'dc1':1, 'dc2':1};"
)
session.execute(
"ALTER KEYSPACE system_auth WITH REPLICATION = {'class':'NetworkTopologyStrategy', 'dc1':1, 'dc2':1};"
)
# Path to byteman script which makes the streaming to node2 throw an exception, making rebuild fail
if cluster.version() < '4.0':
script = [
mk_bman_path('pre4.0/inject_failure_streaming_to_node2.btm')
]
else:
script = [
mk_bman_path('4.0/inject_failure_streaming_to_node2.btm')
]
node3.byteman_submit(script)
# First rebuild must fail and data must be incomplete
with pytest.raises(ToolError, message='Unexpected: SUCCEED'):
logger.debug('Executing first rebuild -> '),
node3.nodetool('rebuild dc1')
logger.debug('Expected: FAILED')
session.execute('USE ks')
with pytest.raises(AssertionError, message='Unexpected: COMPLETE'):
logger.debug('Checking data is complete -> '),
for i in range(0, 20000):
query_c1c2(session, i, ConsistencyLevel.LOCAL_ONE)
logger.debug('Expected: INCOMPLETE')
logger.debug('Executing second rebuild -> '),
node3.nodetool('rebuild dc1')
logger.debug('Expected: SUCCEED')
# Check all streaming sessions completed, streamed ranges are skipped and verify streamed data
node3.watch_log_for('All sessions completed')
node3.watch_log_for('Skipping streaming those ranges.')
logger.debug('Checking data is complete -> '),
for i in range(0, 20000):
query_c1c2(session, i, ConsistencyLevel.LOCAL_ONE)
logger.debug('Expected: COMPLETE')
