def force_test(self):
"""
forcing an incremental repair should incrementally repair any nodes
that are up, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
node2.stop()
# repair should fail because node2 is down
with self.assertRaises(ToolError):
node1.repair(options=['ks'])
# run with force flag
node1.repair(options=['ks', '--force'])
# ... and verify nothing was promoted to repaired
self.assertNoRepairedSSTables(node1, 'ks')
self.assertNoRepairedSSTables(node2, 'ks')
def validate_ssl_options(ssl_options):
# find absolute path to client CA_CERTS
tries = 0
while True:
if tries > 5:
raise RuntimeError("Failed to connect to SSL cluster after 5 attempts")
try:
cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options=ssl_options)
session = cluster.connect()
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def validate_ssl_options(ssl_options):
# find absolute path to client CA_CERTS
tries = 0
while True:
if tries > 5:
raise RuntimeError(
"Failed to connect to SSL cluster after 5 attempts")
try:
cluster = Cluster(protocol_version=PROTOCOL_VERSION,
ssl_options=ssl_options)
session = cluster.connect(wait_for_all_pools=True)
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def test_ssl_connection(self):
"""
Test to validate that we are able to connect to a cluster using ssl.
test_ssl_connection Performs a simple sanity check to ensure that we can connect to a cluster with ssl.
@since 2.6.0
@jira_ticket PYTHON-332
@expected_result we can connect and preform some basic operations
@test_category connection:ssl
"""
# Setup temporary keyspace.
abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS)
self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path,
'ssl_version': ssl.PROTOCOL_TLSv1})
self.session = self.cluster.connect()
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
self.session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
self.session.execute(statement)
def test_can_connect_with_ssl_client_auth(self):
"""
Test to validate that we can connect to a C* cluster that has client_auth enabled.
This test will setup and use a c* cluster that has client authentication enabled. It will then attempt
to connect using valid client keys, and certs (that are in the server's truststore), and attempt to preform some
basic operations
@since 2.7.0
@expected_result The client can connect via SSL and preform some basic operations
@test_category connection:ssl
"""
# Need to get absolute paths for certs/key
abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS)
abs_driver_keyfile = os.path.abspath(DRIVER_KEYFILE)
abs_driver_certfile = os.path.abspath(DRIVER_CERTFILE)
tries = 0
while True:
if tries > 5:
raise RuntimeError(
"Failed to connect to SSL cluster after 5 attempts")
try:
cluster = Cluster(protocol_version=PROTOCOL_VERSION,
ssl_options={
'ca_certs': abs_path_ca_cert_path,
'ssl_version': ssl.PROTOCOL_TLSv1,
'keyfile': abs_driver_keyfile,
'certfile': abs_driver_certfile
})
session = cluster.connect()
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def test_can_connect_with_ssl_ca(self):
"""
Test to validate that we are able to connect to a cluster using ssl.
test_can_connect_with_ssl_ca performs a simple sanity check to ensure that we can connect to a cluster with ssl
authentication via simple server-side shared certificate authority. The client is able to validate the identity
of the server, however by using this method the server can't trust the client unless additional authentication
has been provided.
@since 2.6.0
@jira_ticket PYTHON-332
@expected_result The client can connect via SSL and preform some basic operations
@test_category connection:ssl
"""
# find absolute path to client CA_CERTS
abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS)
tries = 0
while True:
if tries > 5:
raise RuntimeError(
"Failed to connect to SSL cluster after 5 attempts")
try:
cluster = Cluster(protocol_version=PROTOCOL_VERSION,
ssl_options={
'ca_certs': abs_path_ca_cert_path,
'ssl_version': ssl.PROTOCOL_TLSv1
})
session = cluster.connect()
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def test_can_connect_with_ssl_client_auth(self):
"""
Test to validate that we can connect to a C* cluster that has client_auth enabled.
This test will setup and use a c* cluster that has client authentication enabled. It will then attempt
to connect using valid client keys, and certs (that are in the server's truststore), and attempt to preform some
basic operations
@since 2.7.0
@expected_result The client can connect via SSL and preform some basic operations
@test_category connection:ssl
"""
# Need to get absolute paths for certs/key
abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS)
abs_driver_keyfile = os.path.abspath(DRIVER_KEYFILE)
abs_driver_certfile = os.path.abspath(DRIVER_CERTFILE)
tries = 0
while True:
if tries > 5:
raise RuntimeError("Failed to connect to SSL cluster after 5 attempts")
try:
cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path,
'ssl_version': ssl.PROTOCOL_TLSv1,
'keyfile': abs_driver_keyfile,
'certfile': abs_driver_certfile})
session = cluster.connect()
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def test_can_connect_with_ssl_ca(self):
"""
Test to validate that we are able to connect to a cluster using ssl.
test_can_connect_with_ssl_ca performs a simple sanity check to ensure that we can connect to a cluster with ssl
authentication via simple server-side shared certificate authority. The client is able to validate the identity
of the server, however by using this method the server can't trust the client unless additional authentication
has been provided.
@since 2.6.0
@jira_ticket PYTHON-332
@expected_result The client can connect via SSL and preform some basic operations
@test_category connection:ssl
"""
# Setup temporary keyspace.
abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS)
tries = 0
while True:
if tries > 5:
raise RuntimeError("Failed to connect to SSL cluster after 5 attempts")
try:
cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path,
'ssl_version': ssl.PROTOCOL_TLSv1})
session = cluster.connect()
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def _perform_cql_statement(self,
text,
consistency_level,
expected_exception,
session=None):
"""
Simple helper method to preform cql statements and check for expected exception
@param text CQl statement to execute
@param consistency_level Consistency level at which it is to be executed
@param expected_exception Exception expected to be throw or none
"""
if session is None:
session = self.session
statement = SimpleStatement(text)
statement.consistency_level = consistency_level
if expected_exception is None:
self.execute_helper(session, statement)
else:
with self.assertRaises(expected_exception) as cm:
self.execute_helper(session, statement)
if self.support_v5 and (isinstance(cm.exception, WriteFailure)
or isinstance(cm.exception, ReadFailure)):
if isinstance(cm.exception, ReadFailure):
self.assertEqual(
list(cm.exception.error_code_map.values())[0], 1)
else:
self.assertEqual(
list(cm.exception.error_code_map.values())[0], 0)
def test_repaired_tracking_with_partition_deletes(self):
"""
check that when an tracking repaired data status following a digest mismatch,
repaired data mismatches are marked as unconfirmed as we may skip sstables
after the partition delete are encountered.
@jira_ticket CASSANDRA-14145
"""
session, node1, node2 = self.setup_for_repaired_data_tracking()
stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
node1.repair(options=['ks'])
node2.stop(wait_other_notice=True)
session.execute("delete from ks.tbl where k = 5")
node1.flush()
node2.start()
# expect unconfirmed inconsistencies as the partition deletes cause some sstables to be skipped
with JolokiaAgent(node1) as jmx:
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5",
expect_unconfirmed_inconsistencies=True)
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5",
expect_unconfirmed_inconsistencies=True)
# no digest reads for range queries so blocking read repair metric isn't incremented
# *all* sstables are read for partition ranges too, and as the repaired set is still in sync there should
# be no inconsistencies
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def subrange_test(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False,
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500,
'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner'})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
for node in cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
# only repair the partition k=0
token = Murmur3Token.from_key(str(bytearray([0, 0, 0, 0])))
# import ipdb; ipdb.set_trace()
# run with force flag
node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)])
# verify we have a mix of repaired and unrepaired sstables
self.assertRepairedAndUnrepaired(node1, 'ks')
self.assertRepairedAndUnrepaired(node2, 'ks')
self.assertRepairedAndUnrepaired(node3, 'ks')
def test_force_with_none_down(self):
"""
if we force an incremental repair, but all the involved nodes are up,
we should run normally and promote sstables afterwards
"""
self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping({'hinted_handoff_enabled': 'false',
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500})
self.init_default_config()
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
# run with force flag
node1.repair(options=['ks', '--force'])
# ... and verify everything was still promoted
self.assertAllRepairedSSTables(node1, 'ks')
self.assertAllRepairedSSTables(node2, 'ks')
self.assertAllRepairedSSTables(node3, 'ks')
def subrange_test(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False,
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500,
'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner'})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
for node in cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
# only repair the partition k=0
token = Murmur3Token.from_key(str(bytearray([0,0,0,0])))
# import ipdb; ipdb.set_trace()
# run with force flag
node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)])
# verify we have a mix of repaired and unrepaired sstables
self.assertRepairedAndUnrepaired(node1, 'ks')
self.assertRepairedAndUnrepaired(node2, 'ks')
self.assertRepairedAndUnrepaired(node3, 'ks')
def force_test(self):
"""
forcing an incremental repair should incrementally repair any nodes
that are up, but should not promote the sstables to repaired
"""
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
node2.stop()
# repair should fail because node2 is down
with self.assertRaises(ToolError):
node1.repair(options=['ks'])
# run with force flag
node1.repair(options=['ks', '--force'])
# ... and verify nothing was promoted to repaired
self.assertNoRepairedSSTables(node1, 'ks')
self.assertNoRepairedSSTables(node2, 'ks')
def test_force_with_none_down(self):
"""
if we force an incremental repair, but all the involved nodes are up,
we should run normally and promote sstables afterwards
"""
self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping(
{
'hinted_handoff_enabled': 'false',
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500
})
self.init_default_config()
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute(
"CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}"
)
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
# run with force flag
node1.repair(options=['ks', '--force'])
# ... and verify everything was still promoted
self.assertAllRepairedSSTables(node1, 'ks')
self.assertAllRepairedSSTables(node2, 'ks')
self.assertAllRepairedSSTables(node3, 'ks')
def test_hosts(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping(
{
'hinted_handoff_enabled': 'false',
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500
})
self.init_default_config()
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute(
"CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}"
)
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
# run with force flag
node1.repair(options=[
'ks', '-hosts', ','.join([node1.address(),
node2.address()])
])
# ... and verify nothing was promoted to repaired
self.assertNoRepairedSSTables(node1, 'ks')
self.assertNoRepairedSSTables(node2, 'ks')
def test_repaired_tracking_with_partition_deletes(self):
"""
check that when an tracking repaired data status following a digest mismatch,
repaired data mismatches are marked as unconfirmed as we may skip sstables
after the partition delete are encountered.
@jira_ticket CASSANDRA-14145
"""
session, node1, node2 = self.setup_for_repaired_data_tracking()
stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
node1.repair(options=['ks'])
node2.stop(wait_other_notice=True)
session.execute("delete from ks.tbl where k = 5")
node1.flush()
node2.start(wait_other_notice=True)
# expect unconfirmed inconsistencies as the partition deletes cause some sstables to be skipped
with JolokiaAgent(node1) as jmx:
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5",
expect_unconfirmed_inconsistencies=True)
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5",
expect_unconfirmed_inconsistencies=True)
# no digest reads for range queries so blocking read repair metric isn't incremented
# *all* sstables are read for partition ranges too, and as the repaired set is still in sync there should
# be no inconsistencies
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def test_hosts(self):
"""
running an incremental repair with hosts specified should incrementally repair
the given nodes, but should not promote the sstables to repaired
"""
self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping({'hinted_handoff_enabled': 'false',
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500})
self.init_default_config()
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
# run with force flag
node1.repair(options=['ks', '-hosts', ','.join([node1.address(), node2.address()])])
# ... and verify nothing was promoted to repaired
self.assertNoRepairedSSTables(node1, 'ks')
self.assertNoRepairedSSTables(node2, 'ks')
def query_and_check_repaired_mismatches(
self,
jmx,
session,
query,
expect_read_repair=True,
expect_unconfirmed_inconsistencies=False,
expect_confirmed_inconsistencies=False):
rr_count = make_mbean('metrics',
type='ReadRepair',
name='ReconcileRead')
unconfirmed_count = make_mbean(
'metrics',
type='Table,keyspace=ks',
name='RepairedDataInconsistenciesUnconfirmed,scope=tbl')
confirmed_count = make_mbean(
'metrics',
type='Table,keyspace=ks',
name='RepairedDataInconsistenciesConfirmed,scope=tbl')
rr_before = self.get_attribute_count(jmx, rr_count)
uc_before = self.get_attribute_count(jmx, unconfirmed_count)
cc_before = self.get_attribute_count(jmx, confirmed_count)
stmt = SimpleStatement(query)
stmt.consistency_level = ConsistencyLevel.ALL
session.execute(stmt)
rr_after = self.get_attribute_count(jmx, rr_count)
uc_after = self.get_attribute_count(jmx, unconfirmed_count)
cc_after = self.get_attribute_count(jmx, confirmed_count)
logger.debug("Read Repair Count: {before}, {after}".format(
before=rr_before, after=rr_after))
logger.debug(
"Unconfirmed Inconsistency Count: {before}, {after}".format(
before=uc_before, after=uc_after))
logger.debug("Confirmed Inconsistency Count: {before}, {after}".format(
before=cc_before, after=cc_after))
if expect_read_repair:
assert rr_after > rr_before
else:
assert rr_after == rr_before
if expect_unconfirmed_inconsistencies:
assert uc_after > uc_before
else:
assert uc_after == uc_before
if expect_confirmed_inconsistencies:
assert cc_after > cc_before
else:
assert cc_after == cc_before
def test_repaired_tracking_with_mismatching_replicas(self):
"""
verify that when replicas have different repaired sets, this can be detected via the digests
computed at read time. All nodes have start with the same data, but only 1 replica's sstables
are marked repaired. Then a divergence is introduced by overwriting on 1 replica only, which
is required to trigger a digest mismatch & full data read (for single partition reads).
As the repaired sets are different between the replicas, but no other shortcutting occurs
(no partition tombstones or sstable skipping) and no sstables are involved in pending repair
session, we expect confirmed inconsistencies to be reported.
there are two variants of this, for single partition slice & names reads and range reads
@jira_ticket CASSANDRA-14145
"""
session, node1, node2 = self.setup_for_repaired_data_tracking()
stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
for i in range(10,20):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
# stop node 2 and mark its sstables repaired
node2.stop(wait_other_notice=True)
node2.run_sstablerepairedset(keyspace='ks')
# before restarting node2 overwrite some data on node1 to trigger digest mismatches
session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)")
node2.start(wait_for_binary_proto=True)
out1 = node1.run_sstablemetadata(keyspace='ks').stdout
out2 = node2.run_sstablemetadata(keyspace='ks').stdout
# verify the repaired at times for the sstables on node1/node2
assert all(t == 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out1)]])
assert all(t > 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out2)]])
# we expect inconsistencies due to sstables being marked repaired on one replica only
# these are marked confirmed because no sessions are pending & all sstables are
# skipped due to partition deletes
with JolokiaAgent(node1) as jmx:
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5",
expect_confirmed_inconsistencies=True)
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5",
expect_confirmed_inconsistencies=True)
# no digest reads for range queries so read repair metric isn't incremented
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl",
expect_confirmed_inconsistencies=True,
expect_read_repair=False)
def test_repaired_tracking_with_mismatching_replicas(self):
"""
verify that when replicas have different repaired sets, this can be detected via the digests
computed at read time. All nodes have start with the same data, but only 1 replica's sstables
are marked repaired. Then a divergence is introduced by overwriting on 1 replica only, which
is required to trigger a digest mismatch & full data read (for single partition reads).
As the repaired sets are different between the replicas, but no other shortcutting occurs
(no partition tombstones or sstable skipping) and no sstables are involved in pending repair
session, we expect confirmed inconsistencies to be reported.
there are two variants of this, for single partition slice & names reads and range reads
@jira_ticket CASSANDRA-14145
"""
session, node1, node2 = self.setup_for_repaired_data_tracking()
stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
for i in range(10,20):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
# stop node 2 and mark its sstables repaired
node2.stop(wait_other_notice=True)
node2.run_sstablerepairedset(keyspace='ks')
# before restarting node2 overwrite some data on node1 to trigger digest mismatches
session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)")
node2.start(wait_for_binary_proto=True)
out1 = node1.run_sstablemetadata(keyspace='ks').stdout
out2 = node2.run_sstablemetadata(keyspace='ks').stdout
# verify the repaired at times for the sstables on node1/node2
assert all(t == 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out1)]])
assert all(t > 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out2)]])
#Â we expect inconsistencies due to sstables being marked repaired on one replica only
# these are marked confirmed because no sessions are pending & all sstables are
# skipped due to partition deletes
with JolokiaAgent(node1) as jmx:
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5",
expect_confirmed_inconsistencies=True)
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5",
expect_confirmed_inconsistencies=True)
# no digest reads for range queries so read repair metric isn't incremented
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def validate_ssl_options(**kwargs):
ssl_options = kwargs.get('ssl_options', None)
ssl_context = kwargs.get('ssl_context', None)
hostname = kwargs.get('hostname', '127.0.0.1')
# find absolute path to client CA_CERTS
tries = 0
while True:
if tries > 5:
raise RuntimeError(
"Failed to connect to SSL cluster after 5 attempts")
try:
cluster = TestCluster(contact_points=[DefaultEndPoint(hostname)],
ssl_options=ssl_options,
ssl_context=ssl_context)
session = cluster.connect(wait_for_all_pools=True)
break
except Exception:
ex_type, ex, tb = sys.exc_info()
log.warning("{0}: {1} Backtrace: {2}".format(
ex_type.__name__, ex, traceback.extract_tb(tb)))
del tb
tries += 1
# attempt a few simple commands.
insert_keyspace = """CREATE KEYSPACE ssltest
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'}
"""
statement = SimpleStatement(insert_keyspace)
statement.consistency_level = 3
session.execute(statement)
drop_keyspace = "DROP KEYSPACE ssltest"
statement = SimpleStatement(drop_keyspace)
statement.consistency_level = ConsistencyLevel.ANY
session.execute(statement)
cluster.shutdown()
def execute_unlimited_query(stream_key, cols, time_bin, time_range, session=None, prepared=None,
query_consistency=None):
base = ("select %s from %s where subsite=%%s and node=%%s and sensor=%%s and bin=%%s " + \
"and method=%%s and time>=%%s and time<=%%s") % (','.join(cols), stream_key.stream.name)
query = SimpleStatement(base)
query.consistency_level = query_consistency
return list(session.execute(query, (stream_key.subsite,
stream_key.node,
stream_key.sensor,
time_bin,
stream_key.method,
time_range.start,
time_range.stop)))
def test_repaired_tracking_with_varying_sstable_sets(self):
"""
verify that repaired data digests are computed over the merged data for each replica
and that the particular number of sstables on each doesn't affect the comparisons
both replicas start with the same repaired set, comprising 2 sstables. node1's is
then compacted and additional unrepaired data added (which overwrites some in the
repaired set). We expect the repaired digests to still match as the tracking will
force all sstables containing the partitions to be read
there are two variants of this, for single partition slice & names reads and range reads
@jira_ticket CASSANDRA-14145
"""
session, node1, node2 = self.setup_for_repaired_data_tracking()
stmt = SimpleStatement(
"INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
for i in range(10, 20):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
node1.repair(options=['ks'])
node2.stop(wait_other_notice=True)
session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)")
session.execute("insert into ks.tbl (k, c, v) values (15, 15, 155)")
node1.flush()
node1.compact()
node1.compact()
node2.start()
# we don't expect any inconsistencies as all repaired data is read on both replicas
with JolokiaAgent(node1) as jmx:
self.query_and_check_repaired_mismatches(
jmx, session, "SELECT * FROM ks.tbl WHERE k = 5")
self.query_and_check_repaired_mismatches(
jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5")
# no digest reads for range queries so read repair metric isn't incremented
self.query_and_check_repaired_mismatches(jmx,
session,
"SELECT * FROM ks.tbl",
expect_read_repair=False)
def _perform_cql_statement(self, text, consistency_level, expected_exception):
"""
Simple helper method to preform cql statements and check for expected exception
@param text CQl statement to execute
@param consistency_level Consistency level at which it is to be executed
@param expected_exception Exception expected to be throw or none
"""
statement = SimpleStatement(text)
statement.consistency_level = consistency_level
if expected_exception is None:
self.execute_helper(self.session, statement)
else:
with self.assertRaises(expected_exception):
self.execute_helper(self.session, statement)
def _perform_cql_statement(self, text, consistency_level, expected_exception):
"""
Simple helper method to preform cql statements and check for expected exception
@param text CQl statement to execute
@param consistency_level Consistency level at which it is to be executed
@param expected_exception Exception expected to be throw or none
"""
statement = SimpleStatement(text)
statement.consistency_level = consistency_level
if expected_exception is None:
self.execute_helper(self.session, statement)
else:
with self.assertRaises(expected_exception):
self.execute_helper(self.session, statement)
def test_repaired_tracking_with_varying_sstable_sets(self):
"""
verify that repaired data digests are computed over the merged data for each replica
and that the particular number of sstables on each doesn't affect the comparisons
both replicas start with the same repaired set, comprising 2 sstables. node1's is
then compacted and additional unrepaired data added (which overwrites some in the
repaired set). We expect the repaired digests to still match as the tracking will
force all sstables containing the partitions to be read
there are two variants of this, for single partition slice & names reads and range reads
@jira_ticket CASSANDRA-14145
"""
session, node1, node2 = self.setup_for_repaired_data_tracking()
stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
for i in range(10,20):
session.execute(stmt, (i, i, i))
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
node1.repair(options=['ks'])
node2.stop(wait_other_notice=True)
session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)")
session.execute("insert into ks.tbl (k, c, v) values (15, 15, 155)")
node1.flush()
node1.compact()
node1.compact()
node2.start(wait_other_notice=True)
#Â we don't expect any inconsistencies as all repaired data is read on both replicas
with JolokiaAgent(node1) as jmx:
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5")
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5")
# no digest reads for range queries so read repair metric isn't incremented
self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def query_and_check_repaired_mismatches(self, jmx, session, query,
expect_read_repair=True,
expect_unconfirmed_inconsistencies=False,
expect_confirmed_inconsistencies=False):
rr_count = make_mbean('metrics', type='ReadRepair', name='ReconcileRead')
unconfirmed_count = make_mbean('metrics', type='Table,keyspace=ks', name='RepairedDataInconsistenciesUnconfirmed,scope=tbl')
confirmed_count = make_mbean('metrics', type='Table,keyspace=ks', name='RepairedDataInconsistenciesConfirmed,scope=tbl')
rr_before = self.get_attribute_count(jmx, rr_count)
uc_before = self.get_attribute_count(jmx, unconfirmed_count)
cc_before = self.get_attribute_count(jmx, confirmed_count)
stmt = SimpleStatement(query)
stmt.consistency_level = ConsistencyLevel.ALL
session.execute(stmt)
rr_after = self.get_attribute_count(jmx, rr_count)
uc_after = self.get_attribute_count(jmx, unconfirmed_count)
cc_after = self.get_attribute_count(jmx, confirmed_count)
logger.debug("Read Repair Count: {before}, {after}".format(before=rr_before, after=rr_after))
logger.debug("Unconfirmed Inconsistency Count: {before}, {after}".format(before=uc_before, after=uc_after))
logger.debug("Confirmed Inconsistency Count: {before}, {after}".format(before=cc_before, after=cc_after))
if expect_read_repair:
assert rr_after > rr_before
else:
assert rr_after == rr_before
if expect_unconfirmed_inconsistencies:
assert uc_after > uc_before
else:
assert uc_after == uc_before
if expect_confirmed_inconsistencies:
assert cc_after > cc_before
else:
assert cc_after == cc_before
def _perform_cql_statement(self, text, consistency_level, expected_exception, session=None):
"""
Simple helper method to preform cql statements and check for expected exception
@param text CQl statement to execute
@param consistency_level Consistency level at which it is to be executed
@param expected_exception Exception expected to be throw or none
"""
if session is None:
session = self.session
statement = SimpleStatement(text)
statement.consistency_level = consistency_level
if expected_exception is None:
self.execute_helper(session, statement)
else:
with self.assertRaises(expected_exception) as cm:
self.execute_helper(session, statement)
if ProtocolVersion.uses_error_code_map(PROTOCOL_VERSION):
if isinstance(cm.exception, ReadFailure):
self.assertEqual(list(cm.exception.error_code_map.values())[0], 1)
if isinstance(cm.exception, WriteFailure):
self.assertEqual(list(cm.exception.error_code_map.values())[0], 0)
from cassandra.cluster import Cluster
from cassandra.query import SimpleStatement
from cassandra import ConsistencyLevel
cluster = Cluster()
session = cluster.connect("designsite") # or designsite
query = SimpleStatement(
"""INSERT INTO users (login, nickname, password, sites, premium, plan, country, occupation, user_styles)
VALUES (%s, %s, %s, {%s}, %s, %s, %s, %s, {%s});""",
consistency_level=ConsistencyLevel.ONE)
session.execute(query, ('ASIMER', 'ASIMER', 'ASIMER', 'newsmy.com', None, None,
'Ukraine', 'student', 'ancient'))
query.consistency_level = ConsistencyLevel.ONE
session.execute(query, ('Kelioris', 'Kelioris', 'Kelioris', 'newstest.com',
'2020-09-1', 'econom', 'Russia', 'work', 'futuristic'))
query.consistency_level = ConsistencyLevel.ONE
session.execute(
query, ('Tereshchenko', 'Tereshchenko', 'Tereshchenko', 'coolnews.com',
'2020-10-1', 'pro', 'Belarus', 'teacher', 'modern'))
query.consistency_level = ConsistencyLevel.ONE
rows = session.execute("SELECT * FROM users;")
for row in rows:
print(row)
query = SimpleStatement(
"""INSERT INTO topic_analitycs(topic_name, block_type, block_name, words, position, styles, popularity, paragraphs, focus_time, sentences, images)
VALUES (%s, %s, %s, %s, %s, {%s}, %s, %s, %s, %s, %s);""",
consistency_level=ConsistencyLevel.ONE)
session.execute(query, ('comedy', 'div', 'news', 1245, 'center', 'futuristic',
154, 12, '0:1:13.878746', 14, 0.4))
def insert_dataset(stream_key, dataset, session=None, prepared=None, query_consistency=None ):
"""
Insert an xray dataset back into CASSANDRA.
First we check to see if there is data in the bin, if there is we either overwrite and update
the values or fail and let the user known why
:param stream_key: Stream that we are updating
:param dataset: xray dataset we are updating
:return:
"""
# It's easier to use pandas. So covert to dataframe
dataframe = dataset.to_dataframe()
# All of the bins on SAN data will be the same in the netcdf file take the first
data_bin = dataframe['bin'].values[0]
#get the metadata partion
meta = query_partition_metadata(stream_key, TimeRange(bin_to_time(data_bin),
bin_to_time(data_bin + 1)))
bin_meta = None
for i in meta:
if i[0] == data_bin and i[1] == CASS_LOCATION_NAME:
bin_meta = i
break
# get the data in the correct format
cols = get_query_columns(stream_key)
cols = ['subsite', 'node', 'sensor', 'bin', 'method'] + cols[1:]
arrays = set([p.name for p in stream_key.stream.parameters if p.parameter_type != FUNCTION and p.is_array])
dataframe['subsite'] = stream_key.subsite
dataframe['node'] =stream_key.node
dataframe['sensor'] = stream_key.sensor
dataframe['method'] = stream_key.method
# id and provenance are expected to be UUIDs so convert them to uuids
dataframe['id'] = dataframe['id'].apply(lambda x: uuid.UUID(x))
dataframe['provenance'] = dataframe['provenance'].apply(lambda x: uuid.UUID(x))
for i in arrays:
dataframe[i] = dataframe[i].apply(lambda x: msgpack.packb(x))
dataframe = dataframe[cols]
# if we don't have metadata for the bin or we want to overwrite the values from cassandra continue
count = 0
if bin_meta is None or engine.app.config['SAN_CASS_OVERWRITE']:
if bin_meta is not None:
log.warn("Data present in Cassandra bin %s for %s. Overwriting old and adding new data.", data_bin, stream_key.as_refdes())
# get the query to insert information
query_name = 'load_{:s}_{:s}_{:s}_{:s}_{:s}'.format(stream_key.stream_name, stream_key.subsite, stream_key.node, stream_key.sensor, stream_key.method)
if query_name not in prepared:
query = 'INSERT INTO {:s} ({:s}) values ({:s})'.format(stream_key.stream.name,', '.join(cols), ', '.join(['?' for _ in cols]) )
query = session.prepare(query)
query.consistency_level = query_consistency
prepared[query_name] = query
query = prepared[query_name]
# insert data
for good, x in execute_concurrent_with_args(session, query, dataframe.values.tolist(), concurrency=50):
if not good:
log.warn("Failed to insert a particle into Cassandra bin %d for %s!", data_bin, stream_key.as_refdes())
else:
count += 1
else:
# If there is already data and we do not want overwriting return an error
error_message = "Data present in Cassandra bin {:d} for {:s}. Aborting operation!".format(data_bin, stream_key.as_refdes())
log.error(error_message)
return error_message
# Update the metadata
if bin_meta is None:
# Create metadata entry for the new bin
ref_des = stream_key.as_three_part_refdes()
st = dataframe['time'].min()
et = dataframe['time'].max()
meta_query = "INSERT INTO partition_metadata (stream, refdes, method, bin, store, count, first, last) values ('{:s}', '{:s}', '{:s}', {:d}, '{:s}', {:d}, {:f}, {:f})"\
.format(stream_key.stream.name, ref_des, stream_key.method, data_bin, CASS_LOCATION_NAME, count, st, et)
meta_query = SimpleStatement(meta_query)
meta_query.consistency_level = query_consistency
session.execute(meta_query)
ret_val = 'Inserted {:d} particles into Cassandra bin {:d} for {:s}.'.format(count, dataframe['bin'].values[0], stream_key.as_refdes())
log.info(ret_val)
return ret_val
else:
# get the new count, check times, and update metadata
q = "SELECT COUNT(*) from {:s} WHERE subsite = '{:s}' and node = '{:s}' and sensor = '{:s}' and bin = {:d} and method = '{:s}'".format(
stream_key.stream.name, stream_key.subsite, stream_key.node, stream_key.sensor, data_bin, stream_key.method)
q = SimpleStatement(q)
q.consistency_level = query_consistency
new_count = session.execute(q)[0][0]
ref_des = stream_key.as_three_part_refdes()
st = min(dataframe['time'].min(), bin_meta[3])
et = max(dataframe['time'].max(), bin_meta[4])
meta_query = "INSERT INTO partition_metadata (stream, refdes, method, bin, store, count, first, last) values ('{:s}', '{:s}', '{:s}', {:d}, '{:s}', {:d}, {:f}, {:f})" \
.format(stream_key.stream.name, ref_des, stream_key.method, data_bin, CASS_LOCATION_NAME, new_count, st, et)
meta_query = SimpleStatement(meta_query)
meta_query.consistency_level = query_consistency
session.execute(meta_query)
ret_val = 'After updating Cassandra bin {:d} for {:s} there are {:d} particles.'.format(data_bin, stream_key.as_refdes(), new_count)
log.info(ret_val)
return ret_val
def UpdateDocAdmin(self, docid, doctype):
session = self.connection#self.connect()
query = SimpleStatement('UPDATE doc_admin set doctype = %s where docid = %s;',consistency_level= ConsistencyLevel.ONE)
query.consistency_level= ConsistencyLevel.ONE
session.execute(query, (doctype,docid))
def preview_test(self):
""" Test that preview correctly detects out of sync data """
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'commitlog_sync_period_in_ms': 500})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
# everything should be in sync
result = node1.repair(options=['ks', '--preview'])
self.assertIn("Previewed data was in sync", result.stdout)
self.assert_no_repair_history(session)
# make data inconsistent between nodes
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
node3.flush()
time.sleep(1)
node3.stop(gently=False)
stmt.consistency_level = ConsistencyLevel.QUORUM
session = self.exclusive_cql_connection(node1)
for i in range(10):
session.execute(stmt, (i + 10, i + 10))
node1.flush()
time.sleep(1)
node1.stop(gently=False)
node3.start(wait_other_notice=True, wait_for_binary_proto=True)
session = self.exclusive_cql_connection(node2)
for i in range(10):
session.execute(stmt, (i + 20, i + 20))
node1.start(wait_other_notice=True, wait_for_binary_proto=True)
# data should not be in sync for full and unrepaired previews
result = node1.repair(options=['ks', '--preview'])
self.assertIn("Total estimated streaming", result.stdout)
self.assertNotIn("Previewed data was in sync", result.stdout)
result = node1.repair(options=['ks', '--preview', '--full'])
self.assertIn("Total estimated streaming", result.stdout)
self.assertNotIn("Previewed data was in sync", result.stdout)
# repaired data should be in sync anyway
result = node1.repair(options=['ks', '--validate'])
self.assertIn("Repaired data is in sync", result.stdout)
self.assert_no_repair_history(session)
# repair the data...
node1.repair(options=['ks'])
for node in cluster.nodelist():
node.nodetool('compact ks tbl')
# ...and everything should be in sync
result = node1.repair(options=['ks', '--preview'])
self.assertIn("Previewed data was in sync", result.stdout)
result = node1.repair(options=['ks', '--preview', '--full'])
self.assertIn("Previewed data was in sync", result.stdout)
result = node1.repair(options=['ks', '--validate'])
self.assertIn("Repaired data is in sync", result.stdout)
def test_consistent_repair(self):
self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping(
{
'hinted_handoff_enabled': 'false',
'num_tokens': 1,
'commitlog_sync_period_in_ms': 500
})
self.cluster.populate(3).start()
node1, node2, node3 = self.cluster.nodelist()
# make data inconsistent between nodes
session = self.patient_exclusive_cql_connection(node3)
session.execute(
"CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}"
)
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
node3.flush()
time.sleep(1)
node3.stop(gently=False)
stmt.consistency_level = ConsistencyLevel.QUORUM
session = self.exclusive_cql_connection(node1)
for i in range(10):
session.execute(stmt, (i + 10, i + 10))
node1.flush()
time.sleep(1)
node1.stop(gently=False)
node3.start(wait_for_binary_proto=True)
session = self.exclusive_cql_connection(node2)
for i in range(10):
session.execute(stmt, (i + 20, i + 20))
node1.start(wait_for_binary_proto=True)
# flush and check that no sstables are marked repaired
for node in self.cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
session = self.patient_exclusive_cql_connection(node)
results = list(session.execute("SELECT * FROM system.repairs"))
assert len(results) == 0, str(results)
# disable compaction so we can verify sstables are marked pending repair
for node in self.cluster.nodelist():
node.nodetool('disableautocompaction ks tbl')
node1.repair(options=['ks'])
# check that all participating nodes have the repair recorded in their system
# table, that all nodes are listed as participants, and that all sstables are
# (still) marked pending repair
expected_participants = {n.address() for n in self.cluster.nodelist()}
expected_participants_wp = {
n.address_and_port()
for n in self.cluster.nodelist()
}
recorded_pending_ids = set()
for node in self.cluster.nodelist():
session = self.patient_exclusive_cql_connection(node)
results = list(session.execute("SELECT * FROM system.repairs"))
assert len(results) == 1
result = results[0]
assert set(result.participants) == expected_participants
if hasattr(result, "participants_wp"):
assert set(result.participants_wp) == expected_participants_wp
assert result.state, ConsistentState.FINALIZED == "4=FINALIZED"
pending_id = result.parent_id
self.assertAllPendingRepairSSTables(node, 'ks', pending_id)
recorded_pending_ids.add(pending_id)
assert len(recorded_pending_ids) == 1
# sstables are compacted out of pending repair by a compaction
# task, we disabled compaction earlier in the test, so here we
# force the compaction and check that all sstables are promoted
for node in self.cluster.nodelist():
node.nodetool('compact ks tbl')
self.assertAllRepairedSSTables(node, 'ks')
def test_preview(self):
""" Test that preview correctly detects out of sync data """
cluster = self.cluster
cluster.set_configuration_options(values={
'hinted_handoff_enabled': False,
'commitlog_sync_period_in_ms': 500
})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
session = self.patient_exclusive_cql_connection(node3)
session.execute(
"CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}"
)
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
# everything should be in sync
result = node1.repair(options=['ks', '--preview'])
assert "Previewed data was in sync" in result.stdout
assert_no_repair_history(session)
assert preview_failure_count(node1) == 0
# make data inconsistent between nodes
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
node3.flush()
time.sleep(1)
node3.stop(gently=False)
stmt.consistency_level = ConsistencyLevel.QUORUM
session = self.exclusive_cql_connection(node1)
for i in range(10):
session.execute(stmt, (i + 10, i + 10))
node1.flush()
time.sleep(1)
node1.stop(gently=False)
node3.start(wait_for_binary_proto=True)
session = self.exclusive_cql_connection(node2)
for i in range(10):
session.execute(stmt, (i + 20, i + 20))
node1.start(wait_for_binary_proto=True)
# data should not be in sync for full and unrepaired previews
result = node1.repair(options=['ks', '--preview'])
assert "Total estimated streaming" in result.stdout
assert "Previewed data was in sync" not in result.stdout
assert preview_failure_count(node1) == 1
result = node1.repair(options=['ks', '--preview', '--full'])
assert "Total estimated streaming" in result.stdout
assert "Previewed data was in sync" not in result.stdout
assert preview_failure_count(node1) == 2
# repaired data should be in sync anyway
result = node1.repair(options=['ks', '--validate'])
assert "Repaired data is in sync" in result.stdout
assert_no_repair_history(session)
# repair the data...
node1.repair(options=['ks'])
for node in cluster.nodelist():
node.nodetool('compact ks tbl')
# ...and everything should be in sync
result = node1.repair(options=['ks', '--preview'])
assert "Previewed data was in sync" in result.stdout
# data is repaired, previewFailure metric should remain same
assert preview_failure_count(node1) == 2
result = node1.repair(options=['ks', '--preview', '--full'])
assert "Previewed data was in sync" in result.stdout
assert preview_failure_count(node1) == 2
result = node1.repair(options=['ks', '--validate'])
assert "Repaired data is in sync" in result.stdout
assert preview_failure_count(node2) == 0
assert preview_failure_count(node3) == 0
def consistent_repair_test(self):
cluster = self.cluster
cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500})
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
# make data inconsistent between nodes
session = self.patient_exclusive_cql_connection(node3)
session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}")
session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)")
stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)")
stmt.consistency_level = ConsistencyLevel.ALL
for i in range(10):
session.execute(stmt, (i, i))
node3.flush()
time.sleep(1)
node3.stop(gently=False)
stmt.consistency_level = ConsistencyLevel.QUORUM
session = self.exclusive_cql_connection(node1)
for i in range(10):
session.execute(stmt, (i + 10, i + 10))
node1.flush()
time.sleep(1)
node1.stop(gently=False)
node3.start(wait_other_notice=True, wait_for_binary_proto=True)
session = self.exclusive_cql_connection(node2)
for i in range(10):
session.execute(stmt, (i + 20, i + 20))
node1.start(wait_other_notice=True, wait_for_binary_proto=True)
# flush and check that no sstables are marked repaired
for node in cluster.nodelist():
node.flush()
self.assertNoRepairedSSTables(node, 'ks')
session = self.patient_exclusive_cql_connection(node)
results = list(session.execute("SELECT * FROM system.repairs"))
self.assertEqual(len(results), 0, str(results))
# disable compaction so we can verify sstables are marked pending repair
for node in cluster.nodelist():
node.nodetool('disableautocompaction ks tbl')
node1.repair(options=['ks'])
# check that all participating nodes have the repair recorded in their system
# table, that all nodes are listed as participants, and that all sstables are
# (still) marked pending repair
expected_participants = {n.address() for n in cluster.nodelist()}
recorded_pending_ids = set()
for node in cluster.nodelist():
session = self.patient_exclusive_cql_connection(node)
results = list(session.execute("SELECT * FROM system.repairs"))
self.assertEqual(len(results), 1)
result = results[0]
self.assertEqual(set(result.participants), expected_participants)
self.assertEqual(result.state, ConsistentState.FINALIZED, "4=FINALIZED")
pending_id = result.parent_id
self.assertAllPendingRepairSSTables(node, 'ks', pending_id)
recorded_pending_ids.add(pending_id)
self.assertEqual(len(recorded_pending_ids), 1)
# sstables are compacted out of pending repair by a compaction
# task, we disabled compaction earlier in the test, so here we
# force the compaction and check that all sstables are promoted
for node in cluster.nodelist():
node.nodetool('compact ks tbl')
self.assertAllRepairedSSTables(node, 'ks')
source = Cluster(
['192.168.52.1', '192.168.52.2'],
port=9042)
dest = Cluster(
['192.168.58.3', '192.168.58.4'],
port=9042)
FORMAT = "%(levelname)s %(asctime)-15s %(message)s"
logging.basicConfig(format=FORMAT, level=logging.DEBUG, filename='kcopy.log')
src_session = source.connect(keyspace=keyspace)
dst_session = dest.connect(keyspace=keyspace)
select = SimpleStatement('SELECT key, column1, value, TTL(value) FROM your_column_family', fetch_size=500)
select.consistency_level = ConsistencyLevel.QUORUM
insert = dst_session.prepare('INSERT INTO your_column_family (key, column1, value) VALUES (?, ?, ?)')
insert_ttl = dst_session.prepare('INSERT INTO your_column_family (key, column1, value) VALUES (?, ?, ?) USING TTL ?')
insert.consistency_level = ConsistencyLevel.QUORUM
insert_ttl.consistency_level = ConsistencyLevel.QUORUM
reading = True
f = src_session.execute_async(select)
w = None
count = 0
def continue_read():
global reading
if f.has_more_pages:
