def force_test(self): """ forcing an incremental repair should incrementally repair any nodes that are up, but should not promote the sstables to repaired """ cluster = self.cluster cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500}) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) node2.stop() # repair should fail because node2 is down with self.assertRaises(ToolError): node1.repair(options=['ks']) # run with force flag node1.repair(options=['ks', '--force']) # ... and verify nothing was promoted to repaired self.assertNoRepairedSSTables(node1, 'ks') self.assertNoRepairedSSTables(node2, 'ks')
def validate_ssl_options(ssl_options): # find absolute path to client CA_CERTS tries = 0 while True: if tries > 5: raise RuntimeError("Failed to connect to SSL cluster after 5 attempts") try: cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options=ssl_options) session = cluster.connect() break except Exception: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def validate_ssl_options(ssl_options): # find absolute path to client CA_CERTS tries = 0 while True: if tries > 5: raise RuntimeError( "Failed to connect to SSL cluster after 5 attempts") try: cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options=ssl_options) session = cluster.connect(wait_for_all_pools=True) break except Exception: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format( ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def test_ssl_connection(self): """ Test to validate that we are able to connect to a cluster using ssl. test_ssl_connection Performs a simple sanity check to ensure that we can connect to a cluster with ssl. @since 2.6.0 @jira_ticket PYTHON-332 @expected_result we can connect and preform some basic operations @test_category connection:ssl """ # Setup temporary keyspace. abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS) self.cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, 'ssl_version': ssl.PROTOCOL_TLSv1}) self.session = self.cluster.connect() # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 self.session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY self.session.execute(statement)
def test_can_connect_with_ssl_client_auth(self): """ Test to validate that we can connect to a C* cluster that has client_auth enabled. This test will setup and use a c* cluster that has client authentication enabled. It will then attempt to connect using valid client keys, and certs (that are in the server's truststore), and attempt to preform some basic operations @since 2.7.0 @expected_result The client can connect via SSL and preform some basic operations @test_category connection:ssl """ # Need to get absolute paths for certs/key abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS) abs_driver_keyfile = os.path.abspath(DRIVER_KEYFILE) abs_driver_certfile = os.path.abspath(DRIVER_CERTFILE) tries = 0 while True: if tries > 5: raise RuntimeError( "Failed to connect to SSL cluster after 5 attempts") try: cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={ 'ca_certs': abs_path_ca_cert_path, 'ssl_version': ssl.PROTOCOL_TLSv1, 'keyfile': abs_driver_keyfile, 'certfile': abs_driver_certfile }) session = cluster.connect() break except Exception: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format( ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def test_can_connect_with_ssl_ca(self): """ Test to validate that we are able to connect to a cluster using ssl. test_can_connect_with_ssl_ca performs a simple sanity check to ensure that we can connect to a cluster with ssl authentication via simple server-side shared certificate authority. The client is able to validate the identity of the server, however by using this method the server can't trust the client unless additional authentication has been provided. @since 2.6.0 @jira_ticket PYTHON-332 @expected_result The client can connect via SSL and preform some basic operations @test_category connection:ssl """ # find absolute path to client CA_CERTS abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS) tries = 0 while True: if tries > 5: raise RuntimeError( "Failed to connect to SSL cluster after 5 attempts") try: cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={ 'ca_certs': abs_path_ca_cert_path, 'ssl_version': ssl.PROTOCOL_TLSv1 }) session = cluster.connect() break except Exception: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format( ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def test_can_connect_with_ssl_client_auth(self): """ Test to validate that we can connect to a C* cluster that has client_auth enabled. This test will setup and use a c* cluster that has client authentication enabled. It will then attempt to connect using valid client keys, and certs (that are in the server's truststore), and attempt to preform some basic operations @since 2.7.0 @expected_result The client can connect via SSL and preform some basic operations @test_category connection:ssl """ # Need to get absolute paths for certs/key abs_path_ca_cert_path = os.path.abspath(CLIENT_CA_CERTS) abs_driver_keyfile = os.path.abspath(DRIVER_KEYFILE) abs_driver_certfile = os.path.abspath(DRIVER_CERTFILE) tries = 0 while True: if tries > 5: raise RuntimeError("Failed to connect to SSL cluster after 5 attempts") try: cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, 'ssl_version': ssl.PROTOCOL_TLSv1, 'keyfile': abs_driver_keyfile, 'certfile': abs_driver_certfile}) session = cluster.connect() break except Exception: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def test_can_connect_with_ssl_ca(self): """ Test to validate that we are able to connect to a cluster using ssl. test_can_connect_with_ssl_ca performs a simple sanity check to ensure that we can connect to a cluster with ssl authentication via simple server-side shared certificate authority. The client is able to validate the identity of the server, however by using this method the server can't trust the client unless additional authentication has been provided. @since 2.6.0 @jira_ticket PYTHON-332 @expected_result The client can connect via SSL and preform some basic operations @test_category connection:ssl """ # Setup temporary keyspace. abs_path_ca_cert_path = os.path.abspath(DEFAULT_CLIENT_CA_CERTS) tries = 0 while True: if tries > 5: raise RuntimeError("Failed to connect to SSL cluster after 5 attempts") try: cluster = Cluster(protocol_version=PROTOCOL_VERSION, ssl_options={'ca_certs': abs_path_ca_cert_path, 'ssl_version': ssl.PROTOCOL_TLSv1}) session = cluster.connect() break except Exception: ex_type, ex, tb = sys.exc_info() log.warn("{0}: {1} Backtrace: {2}".format(ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def _perform_cql_statement(self, text, consistency_level, expected_exception, session=None): """ Simple helper method to preform cql statements and check for expected exception @param text CQl statement to execute @param consistency_level Consistency level at which it is to be executed @param expected_exception Exception expected to be throw or none """ if session is None: session = self.session statement = SimpleStatement(text) statement.consistency_level = consistency_level if expected_exception is None: self.execute_helper(session, statement) else: with self.assertRaises(expected_exception) as cm: self.execute_helper(session, statement) if self.support_v5 and (isinstance(cm.exception, WriteFailure) or isinstance(cm.exception, ReadFailure)): if isinstance(cm.exception, ReadFailure): self.assertEqual( list(cm.exception.error_code_map.values())[0], 1) else: self.assertEqual( list(cm.exception.error_code_map.values())[0], 0)
def test_repaired_tracking_with_partition_deletes(self): """ check that when an tracking repaired data status following a digest mismatch, repaired data mismatches are marked as unconfirmed as we may skip sstables after the partition delete are encountered. @jira_ticket CASSANDRA-14145 """ session, node1, node2 = self.setup_for_repaired_data_tracking() stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') node1.repair(options=['ks']) node2.stop(wait_other_notice=True) session.execute("delete from ks.tbl where k = 5") node1.flush() node2.start() # expect unconfirmed inconsistencies as the partition deletes cause some sstables to be skipped with JolokiaAgent(node1) as jmx: self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5", expect_unconfirmed_inconsistencies=True) self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5", expect_unconfirmed_inconsistencies=True) # no digest reads for range queries so blocking read repair metric isn't incremented # *all* sstables are read for partition ranges too, and as the repaired set is still in sync there should # be no inconsistencies self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def subrange_test(self): """ running an incremental repair with hosts specified should incrementally repair the given nodes, but should not promote the sstables to repaired """ cluster = self.cluster cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500, 'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner'}) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) for node in cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') # only repair the partition k=0 token = Murmur3Token.from_key(str(bytearray([0, 0, 0, 0]))) # import ipdb; ipdb.set_trace() # run with force flag node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)]) # verify we have a mix of repaired and unrepaired sstables self.assertRepairedAndUnrepaired(node1, 'ks') self.assertRepairedAndUnrepaired(node2, 'ks') self.assertRepairedAndUnrepaired(node3, 'ks')
def test_force_with_none_down(self): """ if we force an incremental repair, but all the involved nodes are up, we should run normally and promote sstables afterwards """ self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping({'hinted_handoff_enabled': 'false', 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500}) self.init_default_config() self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) # run with force flag node1.repair(options=['ks', '--force']) # ... and verify everything was still promoted self.assertAllRepairedSSTables(node1, 'ks') self.assertAllRepairedSSTables(node2, 'ks') self.assertAllRepairedSSTables(node3, 'ks')
def subrange_test(self): """ running an incremental repair with hosts specified should incrementally repair the given nodes, but should not promote the sstables to repaired """ cluster = self.cluster cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500, 'partitioner': 'org.apache.cassandra.dht.Murmur3Partitioner'}) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) for node in cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') # only repair the partition k=0 token = Murmur3Token.from_key(str(bytearray([0,0,0,0]))) # import ipdb; ipdb.set_trace() # run with force flag node1.repair(options=['ks', '-st', str(token.value - 1), '-et', str(token.value)]) # verify we have a mix of repaired and unrepaired sstables self.assertRepairedAndUnrepaired(node1, 'ks') self.assertRepairedAndUnrepaired(node2, 'ks') self.assertRepairedAndUnrepaired(node3, 'ks')
def test_force_with_none_down(self): """ if we force an incremental repair, but all the involved nodes are up, we should run normally and promote sstables afterwards """ self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping( { 'hinted_handoff_enabled': 'false', 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500 }) self.init_default_config() self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute( "CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}" ) session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) # run with force flag node1.repair(options=['ks', '--force']) # ... and verify everything was still promoted self.assertAllRepairedSSTables(node1, 'ks') self.assertAllRepairedSSTables(node2, 'ks') self.assertAllRepairedSSTables(node3, 'ks')
def test_hosts(self): """ running an incremental repair with hosts specified should incrementally repair the given nodes, but should not promote the sstables to repaired """ self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping( { 'hinted_handoff_enabled': 'false', 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500 }) self.init_default_config() self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute( "CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}" ) session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) # run with force flag node1.repair(options=[ 'ks', '-hosts', ','.join([node1.address(), node2.address()]) ]) # ... and verify nothing was promoted to repaired self.assertNoRepairedSSTables(node1, 'ks') self.assertNoRepairedSSTables(node2, 'ks')
def test_repaired_tracking_with_partition_deletes(self): """ check that when an tracking repaired data status following a digest mismatch, repaired data mismatches are marked as unconfirmed as we may skip sstables after the partition delete are encountered. @jira_ticket CASSANDRA-14145 """ session, node1, node2 = self.setup_for_repaired_data_tracking() stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') node1.repair(options=['ks']) node2.stop(wait_other_notice=True) session.execute("delete from ks.tbl where k = 5") node1.flush() node2.start(wait_other_notice=True) # expect unconfirmed inconsistencies as the partition deletes cause some sstables to be skipped with JolokiaAgent(node1) as jmx: self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5", expect_unconfirmed_inconsistencies=True) self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5", expect_unconfirmed_inconsistencies=True) # no digest reads for range queries so blocking read repair metric isn't incremented # *all* sstables are read for partition ranges too, and as the repaired set is still in sync there should # be no inconsistencies self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def test_hosts(self): """ running an incremental repair with hosts specified should incrementally repair the given nodes, but should not promote the sstables to repaired """ self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping({'hinted_handoff_enabled': 'false', 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500}) self.init_default_config() self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) # run with force flag node1.repair(options=['ks', '-hosts', ','.join([node1.address(), node2.address()])]) # ... and verify nothing was promoted to repaired self.assertNoRepairedSSTables(node1, 'ks') self.assertNoRepairedSSTables(node2, 'ks')
def query_and_check_repaired_mismatches( self, jmx, session, query, expect_read_repair=True, expect_unconfirmed_inconsistencies=False, expect_confirmed_inconsistencies=False): rr_count = make_mbean('metrics', type='ReadRepair', name='ReconcileRead') unconfirmed_count = make_mbean( 'metrics', type='Table,keyspace=ks', name='RepairedDataInconsistenciesUnconfirmed,scope=tbl') confirmed_count = make_mbean( 'metrics', type='Table,keyspace=ks', name='RepairedDataInconsistenciesConfirmed,scope=tbl') rr_before = self.get_attribute_count(jmx, rr_count) uc_before = self.get_attribute_count(jmx, unconfirmed_count) cc_before = self.get_attribute_count(jmx, confirmed_count) stmt = SimpleStatement(query) stmt.consistency_level = ConsistencyLevel.ALL session.execute(stmt) rr_after = self.get_attribute_count(jmx, rr_count) uc_after = self.get_attribute_count(jmx, unconfirmed_count) cc_after = self.get_attribute_count(jmx, confirmed_count) logger.debug("Read Repair Count: {before}, {after}".format( before=rr_before, after=rr_after)) logger.debug( "Unconfirmed Inconsistency Count: {before}, {after}".format( before=uc_before, after=uc_after)) logger.debug("Confirmed Inconsistency Count: {before}, {after}".format( before=cc_before, after=cc_after)) if expect_read_repair: assert rr_after > rr_before else: assert rr_after == rr_before if expect_unconfirmed_inconsistencies: assert uc_after > uc_before else: assert uc_after == uc_before if expect_confirmed_inconsistencies: assert cc_after > cc_before else: assert cc_after == cc_before
def test_repaired_tracking_with_mismatching_replicas(self): """ verify that when replicas have different repaired sets, this can be detected via the digests computed at read time. All nodes have start with the same data, but only 1 replica's sstables are marked repaired. Then a divergence is introduced by overwriting on 1 replica only, which is required to trigger a digest mismatch & full data read (for single partition reads). As the repaired sets are different between the replicas, but no other shortcutting occurs (no partition tombstones or sstable skipping) and no sstables are involved in pending repair session, we expect confirmed inconsistencies to be reported. there are two variants of this, for single partition slice & names reads and range reads @jira_ticket CASSANDRA-14145 """ session, node1, node2 = self.setup_for_repaired_data_tracking() stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() for i in range(10,20): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') # stop node 2 and mark its sstables repaired node2.stop(wait_other_notice=True) node2.run_sstablerepairedset(keyspace='ks') # before restarting node2 overwrite some data on node1 to trigger digest mismatches session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)") node2.start(wait_for_binary_proto=True) out1 = node1.run_sstablemetadata(keyspace='ks').stdout out2 = node2.run_sstablemetadata(keyspace='ks').stdout # verify the repaired at times for the sstables on node1/node2 assert all(t == 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out1)]]) assert all(t > 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out2)]]) # we expect inconsistencies due to sstables being marked repaired on one replica only # these are marked confirmed because no sessions are pending & all sstables are # skipped due to partition deletes with JolokiaAgent(node1) as jmx: self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5", expect_confirmed_inconsistencies=True) self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5", expect_confirmed_inconsistencies=True) # no digest reads for range queries so read repair metric isn't incremented self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_confirmed_inconsistencies=True, expect_read_repair=False)
def test_repaired_tracking_with_mismatching_replicas(self): """ verify that when replicas have different repaired sets, this can be detected via the digests computed at read time. All nodes have start with the same data, but only 1 replica's sstables are marked repaired. Then a divergence is introduced by overwriting on 1 replica only, which is required to trigger a digest mismatch & full data read (for single partition reads). As the repaired sets are different between the replicas, but no other shortcutting occurs (no partition tombstones or sstable skipping) and no sstables are involved in pending repair session, we expect confirmed inconsistencies to be reported. there are two variants of this, for single partition slice & names reads and range reads @jira_ticket CASSANDRA-14145 """ session, node1, node2 = self.setup_for_repaired_data_tracking() stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() for i in range(10,20): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') # stop node 2 and mark its sstables repaired node2.stop(wait_other_notice=True) node2.run_sstablerepairedset(keyspace='ks') # before restarting node2 overwrite some data on node1 to trigger digest mismatches session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)") node2.start(wait_for_binary_proto=True) out1 = node1.run_sstablemetadata(keyspace='ks').stdout out2 = node2.run_sstablemetadata(keyspace='ks').stdout # verify the repaired at times for the sstables on node1/node2 assert all(t == 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out1)]]) assert all(t > 0 for t in [int(x) for x in [y.split(' ')[0] for y in findall('(?<=Repaired at: ).*', out2)]]) # we expect inconsistencies due to sstables being marked repaired on one replica only # these are marked confirmed because no sessions are pending & all sstables are # skipped due to partition deletes with JolokiaAgent(node1) as jmx: self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5", expect_confirmed_inconsistencies=True) self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5", expect_confirmed_inconsistencies=True) # no digest reads for range queries so read repair metric isn't incremented self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def validate_ssl_options(**kwargs): ssl_options = kwargs.get('ssl_options', None) ssl_context = kwargs.get('ssl_context', None) hostname = kwargs.get('hostname', '127.0.0.1') # find absolute path to client CA_CERTS tries = 0 while True: if tries > 5: raise RuntimeError( "Failed to connect to SSL cluster after 5 attempts") try: cluster = TestCluster(contact_points=[DefaultEndPoint(hostname)], ssl_options=ssl_options, ssl_context=ssl_context) session = cluster.connect(wait_for_all_pools=True) break except Exception: ex_type, ex, tb = sys.exc_info() log.warning("{0}: {1} Backtrace: {2}".format( ex_type.__name__, ex, traceback.extract_tb(tb))) del tb tries += 1 # attempt a few simple commands. insert_keyspace = """CREATE KEYSPACE ssltest WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '3'} """ statement = SimpleStatement(insert_keyspace) statement.consistency_level = 3 session.execute(statement) drop_keyspace = "DROP KEYSPACE ssltest" statement = SimpleStatement(drop_keyspace) statement.consistency_level = ConsistencyLevel.ANY session.execute(statement) cluster.shutdown()
def execute_unlimited_query(stream_key, cols, time_bin, time_range, session=None, prepared=None, query_consistency=None): base = ("select %s from %s where subsite=%%s and node=%%s and sensor=%%s and bin=%%s " + \ "and method=%%s and time>=%%s and time<=%%s") % (','.join(cols), stream_key.stream.name) query = SimpleStatement(base) query.consistency_level = query_consistency return list(session.execute(query, (stream_key.subsite, stream_key.node, stream_key.sensor, time_bin, stream_key.method, time_range.start, time_range.stop)))
def test_repaired_tracking_with_varying_sstable_sets(self): """ verify that repaired data digests are computed over the merged data for each replica and that the particular number of sstables on each doesn't affect the comparisons both replicas start with the same repaired set, comprising 2 sstables. node1's is then compacted and additional unrepaired data added (which overwrites some in the repaired set). We expect the repaired digests to still match as the tracking will force all sstables containing the partitions to be read there are two variants of this, for single partition slice & names reads and range reads @jira_ticket CASSANDRA-14145 """ session, node1, node2 = self.setup_for_repaired_data_tracking() stmt = SimpleStatement( "INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() for i in range(10, 20): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') node1.repair(options=['ks']) node2.stop(wait_other_notice=True) session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)") session.execute("insert into ks.tbl (k, c, v) values (15, 15, 155)") node1.flush() node1.compact() node1.compact() node2.start() # we don't expect any inconsistencies as all repaired data is read on both replicas with JolokiaAgent(node1) as jmx: self.query_and_check_repaired_mismatches( jmx, session, "SELECT * FROM ks.tbl WHERE k = 5") self.query_and_check_repaired_mismatches( jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5") # no digest reads for range queries so read repair metric isn't incremented self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def _perform_cql_statement(self, text, consistency_level, expected_exception): """ Simple helper method to preform cql statements and check for expected exception @param text CQl statement to execute @param consistency_level Consistency level at which it is to be executed @param expected_exception Exception expected to be throw or none """ statement = SimpleStatement(text) statement.consistency_level = consistency_level if expected_exception is None: self.execute_helper(self.session, statement) else: with self.assertRaises(expected_exception): self.execute_helper(self.session, statement)
def test_repaired_tracking_with_varying_sstable_sets(self): """ verify that repaired data digests are computed over the merged data for each replica and that the particular number of sstables on each doesn't affect the comparisons both replicas start with the same repaired set, comprising 2 sstables. node1's is then compacted and additional unrepaired data added (which overwrites some in the repaired set). We expect the repaired digests to still match as the tracking will force all sstables containing the partitions to be read there are two variants of this, for single partition slice & names reads and range reads @jira_ticket CASSANDRA-14145 """ session, node1, node2 = self.setup_for_repaired_data_tracking() stmt = SimpleStatement("INSERT INTO ks.tbl (k, c, v) VALUES (%s, %s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() for i in range(10,20): session.execute(stmt, (i, i, i)) for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') node1.repair(options=['ks']) node2.stop(wait_other_notice=True) session.execute("insert into ks.tbl (k, c, v) values (5, 5, 55)") session.execute("insert into ks.tbl (k, c, v) values (15, 15, 155)") node1.flush() node1.compact() node1.compact() node2.start(wait_other_notice=True) # we don't expect any inconsistencies as all repaired data is read on both replicas with JolokiaAgent(node1) as jmx: self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5") self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl WHERE k = 5 AND c = 5") # no digest reads for range queries so read repair metric isn't incremented self.query_and_check_repaired_mismatches(jmx, session, "SELECT * FROM ks.tbl", expect_read_repair=False)
def query_and_check_repaired_mismatches(self, jmx, session, query, expect_read_repair=True, expect_unconfirmed_inconsistencies=False, expect_confirmed_inconsistencies=False): rr_count = make_mbean('metrics', type='ReadRepair', name='ReconcileRead') unconfirmed_count = make_mbean('metrics', type='Table,keyspace=ks', name='RepairedDataInconsistenciesUnconfirmed,scope=tbl') confirmed_count = make_mbean('metrics', type='Table,keyspace=ks', name='RepairedDataInconsistenciesConfirmed,scope=tbl') rr_before = self.get_attribute_count(jmx, rr_count) uc_before = self.get_attribute_count(jmx, unconfirmed_count) cc_before = self.get_attribute_count(jmx, confirmed_count) stmt = SimpleStatement(query) stmt.consistency_level = ConsistencyLevel.ALL session.execute(stmt) rr_after = self.get_attribute_count(jmx, rr_count) uc_after = self.get_attribute_count(jmx, unconfirmed_count) cc_after = self.get_attribute_count(jmx, confirmed_count) logger.debug("Read Repair Count: {before}, {after}".format(before=rr_before, after=rr_after)) logger.debug("Unconfirmed Inconsistency Count: {before}, {after}".format(before=uc_before, after=uc_after)) logger.debug("Confirmed Inconsistency Count: {before}, {after}".format(before=cc_before, after=cc_after)) if expect_read_repair: assert rr_after > rr_before else: assert rr_after == rr_before if expect_unconfirmed_inconsistencies: assert uc_after > uc_before else: assert uc_after == uc_before if expect_confirmed_inconsistencies: assert cc_after > cc_before else: assert cc_after == cc_before
def _perform_cql_statement(self, text, consistency_level, expected_exception, session=None): """ Simple helper method to preform cql statements and check for expected exception @param text CQl statement to execute @param consistency_level Consistency level at which it is to be executed @param expected_exception Exception expected to be throw or none """ if session is None: session = self.session statement = SimpleStatement(text) statement.consistency_level = consistency_level if expected_exception is None: self.execute_helper(session, statement) else: with self.assertRaises(expected_exception) as cm: self.execute_helper(session, statement) if ProtocolVersion.uses_error_code_map(PROTOCOL_VERSION): if isinstance(cm.exception, ReadFailure): self.assertEqual(list(cm.exception.error_code_map.values())[0], 1) if isinstance(cm.exception, WriteFailure): self.assertEqual(list(cm.exception.error_code_map.values())[0], 0)
from cassandra.cluster import Cluster from cassandra.query import SimpleStatement from cassandra import ConsistencyLevel cluster = Cluster() session = cluster.connect("designsite") # or designsite query = SimpleStatement( """INSERT INTO users (login, nickname, password, sites, premium, plan, country, occupation, user_styles) VALUES (%s, %s, %s, {%s}, %s, %s, %s, %s, {%s});""", consistency_level=ConsistencyLevel.ONE) session.execute(query, ('ASIMER', 'ASIMER', 'ASIMER', 'newsmy.com', None, None, 'Ukraine', 'student', 'ancient')) query.consistency_level = ConsistencyLevel.ONE session.execute(query, ('Kelioris', 'Kelioris', 'Kelioris', 'newstest.com', '2020-09-1', 'econom', 'Russia', 'work', 'futuristic')) query.consistency_level = ConsistencyLevel.ONE session.execute( query, ('Tereshchenko', 'Tereshchenko', 'Tereshchenko', 'coolnews.com', '2020-10-1', 'pro', 'Belarus', 'teacher', 'modern')) query.consistency_level = ConsistencyLevel.ONE rows = session.execute("SELECT * FROM users;") for row in rows: print(row) query = SimpleStatement( """INSERT INTO topic_analitycs(topic_name, block_type, block_name, words, position, styles, popularity, paragraphs, focus_time, sentences, images) VALUES (%s, %s, %s, %s, %s, {%s}, %s, %s, %s, %s, %s);""", consistency_level=ConsistencyLevel.ONE) session.execute(query, ('comedy', 'div', 'news', 1245, 'center', 'futuristic', 154, 12, '0:1:13.878746', 14, 0.4))
def insert_dataset(stream_key, dataset, session=None, prepared=None, query_consistency=None ): """ Insert an xray dataset back into CASSANDRA. First we check to see if there is data in the bin, if there is we either overwrite and update the values or fail and let the user known why :param stream_key: Stream that we are updating :param dataset: xray dataset we are updating :return: """ # It's easier to use pandas. So covert to dataframe dataframe = dataset.to_dataframe() # All of the bins on SAN data will be the same in the netcdf file take the first data_bin = dataframe['bin'].values[0] #get the metadata partion meta = query_partition_metadata(stream_key, TimeRange(bin_to_time(data_bin), bin_to_time(data_bin + 1))) bin_meta = None for i in meta: if i[0] == data_bin and i[1] == CASS_LOCATION_NAME: bin_meta = i break # get the data in the correct format cols = get_query_columns(stream_key) cols = ['subsite', 'node', 'sensor', 'bin', 'method'] + cols[1:] arrays = set([p.name for p in stream_key.stream.parameters if p.parameter_type != FUNCTION and p.is_array]) dataframe['subsite'] = stream_key.subsite dataframe['node'] =stream_key.node dataframe['sensor'] = stream_key.sensor dataframe['method'] = stream_key.method # id and provenance are expected to be UUIDs so convert them to uuids dataframe['id'] = dataframe['id'].apply(lambda x: uuid.UUID(x)) dataframe['provenance'] = dataframe['provenance'].apply(lambda x: uuid.UUID(x)) for i in arrays: dataframe[i] = dataframe[i].apply(lambda x: msgpack.packb(x)) dataframe = dataframe[cols] # if we don't have metadata for the bin or we want to overwrite the values from cassandra continue count = 0 if bin_meta is None or engine.app.config['SAN_CASS_OVERWRITE']: if bin_meta is not None: log.warn("Data present in Cassandra bin %s for %s. Overwriting old and adding new data.", data_bin, stream_key.as_refdes()) # get the query to insert information query_name = 'load_{:s}_{:s}_{:s}_{:s}_{:s}'.format(stream_key.stream_name, stream_key.subsite, stream_key.node, stream_key.sensor, stream_key.method) if query_name not in prepared: query = 'INSERT INTO {:s} ({:s}) values ({:s})'.format(stream_key.stream.name,', '.join(cols), ', '.join(['?' for _ in cols]) ) query = session.prepare(query) query.consistency_level = query_consistency prepared[query_name] = query query = prepared[query_name] # insert data for good, x in execute_concurrent_with_args(session, query, dataframe.values.tolist(), concurrency=50): if not good: log.warn("Failed to insert a particle into Cassandra bin %d for %s!", data_bin, stream_key.as_refdes()) else: count += 1 else: # If there is already data and we do not want overwriting return an error error_message = "Data present in Cassandra bin {:d} for {:s}. Aborting operation!".format(data_bin, stream_key.as_refdes()) log.error(error_message) return error_message # Update the metadata if bin_meta is None: # Create metadata entry for the new bin ref_des = stream_key.as_three_part_refdes() st = dataframe['time'].min() et = dataframe['time'].max() meta_query = "INSERT INTO partition_metadata (stream, refdes, method, bin, store, count, first, last) values ('{:s}', '{:s}', '{:s}', {:d}, '{:s}', {:d}, {:f}, {:f})"\ .format(stream_key.stream.name, ref_des, stream_key.method, data_bin, CASS_LOCATION_NAME, count, st, et) meta_query = SimpleStatement(meta_query) meta_query.consistency_level = query_consistency session.execute(meta_query) ret_val = 'Inserted {:d} particles into Cassandra bin {:d} for {:s}.'.format(count, dataframe['bin'].values[0], stream_key.as_refdes()) log.info(ret_val) return ret_val else: # get the new count, check times, and update metadata q = "SELECT COUNT(*) from {:s} WHERE subsite = '{:s}' and node = '{:s}' and sensor = '{:s}' and bin = {:d} and method = '{:s}'".format( stream_key.stream.name, stream_key.subsite, stream_key.node, stream_key.sensor, data_bin, stream_key.method) q = SimpleStatement(q) q.consistency_level = query_consistency new_count = session.execute(q)[0][0] ref_des = stream_key.as_three_part_refdes() st = min(dataframe['time'].min(), bin_meta[3]) et = max(dataframe['time'].max(), bin_meta[4]) meta_query = "INSERT INTO partition_metadata (stream, refdes, method, bin, store, count, first, last) values ('{:s}', '{:s}', '{:s}', {:d}, '{:s}', {:d}, {:f}, {:f})" \ .format(stream_key.stream.name, ref_des, stream_key.method, data_bin, CASS_LOCATION_NAME, new_count, st, et) meta_query = SimpleStatement(meta_query) meta_query.consistency_level = query_consistency session.execute(meta_query) ret_val = 'After updating Cassandra bin {:d} for {:s} there are {:d} particles.'.format(data_bin, stream_key.as_refdes(), new_count) log.info(ret_val) return ret_val
def UpdateDocAdmin(self, docid, doctype): session = self.connection#self.connect() query = SimpleStatement('UPDATE doc_admin set doctype = %s where docid = %s;',consistency_level= ConsistencyLevel.ONE) query.consistency_level= ConsistencyLevel.ONE session.execute(query, (doctype,docid))
def preview_test(self): """ Test that preview correctly detects out of sync data """ cluster = self.cluster cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'commitlog_sync_period_in_ms': 500}) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") # everything should be in sync result = node1.repair(options=['ks', '--preview']) self.assertIn("Previewed data was in sync", result.stdout) self.assert_no_repair_history(session) # make data inconsistent between nodes stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) node3.flush() time.sleep(1) node3.stop(gently=False) stmt.consistency_level = ConsistencyLevel.QUORUM session = self.exclusive_cql_connection(node1) for i in range(10): session.execute(stmt, (i + 10, i + 10)) node1.flush() time.sleep(1) node1.stop(gently=False) node3.start(wait_other_notice=True, wait_for_binary_proto=True) session = self.exclusive_cql_connection(node2) for i in range(10): session.execute(stmt, (i + 20, i + 20)) node1.start(wait_other_notice=True, wait_for_binary_proto=True) # data should not be in sync for full and unrepaired previews result = node1.repair(options=['ks', '--preview']) self.assertIn("Total estimated streaming", result.stdout) self.assertNotIn("Previewed data was in sync", result.stdout) result = node1.repair(options=['ks', '--preview', '--full']) self.assertIn("Total estimated streaming", result.stdout) self.assertNotIn("Previewed data was in sync", result.stdout) # repaired data should be in sync anyway result = node1.repair(options=['ks', '--validate']) self.assertIn("Repaired data is in sync", result.stdout) self.assert_no_repair_history(session) # repair the data... node1.repair(options=['ks']) for node in cluster.nodelist(): node.nodetool('compact ks tbl') # ...and everything should be in sync result = node1.repair(options=['ks', '--preview']) self.assertIn("Previewed data was in sync", result.stdout) result = node1.repair(options=['ks', '--preview', '--full']) self.assertIn("Previewed data was in sync", result.stdout) result = node1.repair(options=['ks', '--validate']) self.assertIn("Repaired data is in sync", result.stdout)
def test_consistent_repair(self): self.fixture_dtest_setup.setup_overrides.cluster_options = ImmutableMapping( { 'hinted_handoff_enabled': 'false', 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500 }) self.cluster.populate(3).start() node1, node2, node3 = self.cluster.nodelist() # make data inconsistent between nodes session = self.patient_exclusive_cql_connection(node3) session.execute( "CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}" ) session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) node3.flush() time.sleep(1) node3.stop(gently=False) stmt.consistency_level = ConsistencyLevel.QUORUM session = self.exclusive_cql_connection(node1) for i in range(10): session.execute(stmt, (i + 10, i + 10)) node1.flush() time.sleep(1) node1.stop(gently=False) node3.start(wait_for_binary_proto=True) session = self.exclusive_cql_connection(node2) for i in range(10): session.execute(stmt, (i + 20, i + 20)) node1.start(wait_for_binary_proto=True) # flush and check that no sstables are marked repaired for node in self.cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') session = self.patient_exclusive_cql_connection(node) results = list(session.execute("SELECT * FROM system.repairs")) assert len(results) == 0, str(results) # disable compaction so we can verify sstables are marked pending repair for node in self.cluster.nodelist(): node.nodetool('disableautocompaction ks tbl') node1.repair(options=['ks']) # check that all participating nodes have the repair recorded in their system # table, that all nodes are listed as participants, and that all sstables are # (still) marked pending repair expected_participants = {n.address() for n in self.cluster.nodelist()} expected_participants_wp = { n.address_and_port() for n in self.cluster.nodelist() } recorded_pending_ids = set() for node in self.cluster.nodelist(): session = self.patient_exclusive_cql_connection(node) results = list(session.execute("SELECT * FROM system.repairs")) assert len(results) == 1 result = results[0] assert set(result.participants) == expected_participants if hasattr(result, "participants_wp"): assert set(result.participants_wp) == expected_participants_wp assert result.state, ConsistentState.FINALIZED == "4=FINALIZED" pending_id = result.parent_id self.assertAllPendingRepairSSTables(node, 'ks', pending_id) recorded_pending_ids.add(pending_id) assert len(recorded_pending_ids) == 1 # sstables are compacted out of pending repair by a compaction # task, we disabled compaction earlier in the test, so here we # force the compaction and check that all sstables are promoted for node in self.cluster.nodelist(): node.nodetool('compact ks tbl') self.assertAllRepairedSSTables(node, 'ks')
def test_preview(self): """ Test that preview correctly detects out of sync data """ cluster = self.cluster cluster.set_configuration_options(values={ 'hinted_handoff_enabled': False, 'commitlog_sync_period_in_ms': 500 }) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() session = self.patient_exclusive_cql_connection(node3) session.execute( "CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}" ) session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") # everything should be in sync result = node1.repair(options=['ks', '--preview']) assert "Previewed data was in sync" in result.stdout assert_no_repair_history(session) assert preview_failure_count(node1) == 0 # make data inconsistent between nodes stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) node3.flush() time.sleep(1) node3.stop(gently=False) stmt.consistency_level = ConsistencyLevel.QUORUM session = self.exclusive_cql_connection(node1) for i in range(10): session.execute(stmt, (i + 10, i + 10)) node1.flush() time.sleep(1) node1.stop(gently=False) node3.start(wait_for_binary_proto=True) session = self.exclusive_cql_connection(node2) for i in range(10): session.execute(stmt, (i + 20, i + 20)) node1.start(wait_for_binary_proto=True) # data should not be in sync for full and unrepaired previews result = node1.repair(options=['ks', '--preview']) assert "Total estimated streaming" in result.stdout assert "Previewed data was in sync" not in result.stdout assert preview_failure_count(node1) == 1 result = node1.repair(options=['ks', '--preview', '--full']) assert "Total estimated streaming" in result.stdout assert "Previewed data was in sync" not in result.stdout assert preview_failure_count(node1) == 2 # repaired data should be in sync anyway result = node1.repair(options=['ks', '--validate']) assert "Repaired data is in sync" in result.stdout assert_no_repair_history(session) # repair the data... node1.repair(options=['ks']) for node in cluster.nodelist(): node.nodetool('compact ks tbl') # ...and everything should be in sync result = node1.repair(options=['ks', '--preview']) assert "Previewed data was in sync" in result.stdout # data is repaired, previewFailure metric should remain same assert preview_failure_count(node1) == 2 result = node1.repair(options=['ks', '--preview', '--full']) assert "Previewed data was in sync" in result.stdout assert preview_failure_count(node1) == 2 result = node1.repair(options=['ks', '--validate']) assert "Repaired data is in sync" in result.stdout assert preview_failure_count(node2) == 0 assert preview_failure_count(node3) == 0
def consistent_repair_test(self): cluster = self.cluster cluster.set_configuration_options(values={'hinted_handoff_enabled': False, 'num_tokens': 1, 'commitlog_sync_period_in_ms': 500}) cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() # make data inconsistent between nodes session = self.patient_exclusive_cql_connection(node3) session.execute("CREATE KEYSPACE ks WITH REPLICATION={'class':'SimpleStrategy', 'replication_factor': 3}") session.execute("CREATE TABLE ks.tbl (k INT PRIMARY KEY, v INT)") stmt = SimpleStatement("INSERT INTO ks.tbl (k,v) VALUES (%s, %s)") stmt.consistency_level = ConsistencyLevel.ALL for i in range(10): session.execute(stmt, (i, i)) node3.flush() time.sleep(1) node3.stop(gently=False) stmt.consistency_level = ConsistencyLevel.QUORUM session = self.exclusive_cql_connection(node1) for i in range(10): session.execute(stmt, (i + 10, i + 10)) node1.flush() time.sleep(1) node1.stop(gently=False) node3.start(wait_other_notice=True, wait_for_binary_proto=True) session = self.exclusive_cql_connection(node2) for i in range(10): session.execute(stmt, (i + 20, i + 20)) node1.start(wait_other_notice=True, wait_for_binary_proto=True) # flush and check that no sstables are marked repaired for node in cluster.nodelist(): node.flush() self.assertNoRepairedSSTables(node, 'ks') session = self.patient_exclusive_cql_connection(node) results = list(session.execute("SELECT * FROM system.repairs")) self.assertEqual(len(results), 0, str(results)) # disable compaction so we can verify sstables are marked pending repair for node in cluster.nodelist(): node.nodetool('disableautocompaction ks tbl') node1.repair(options=['ks']) # check that all participating nodes have the repair recorded in their system # table, that all nodes are listed as participants, and that all sstables are # (still) marked pending repair expected_participants = {n.address() for n in cluster.nodelist()} recorded_pending_ids = set() for node in cluster.nodelist(): session = self.patient_exclusive_cql_connection(node) results = list(session.execute("SELECT * FROM system.repairs")) self.assertEqual(len(results), 1) result = results[0] self.assertEqual(set(result.participants), expected_participants) self.assertEqual(result.state, ConsistentState.FINALIZED, "4=FINALIZED") pending_id = result.parent_id self.assertAllPendingRepairSSTables(node, 'ks', pending_id) recorded_pending_ids.add(pending_id) self.assertEqual(len(recorded_pending_ids), 1) # sstables are compacted out of pending repair by a compaction # task, we disabled compaction earlier in the test, so here we # force the compaction and check that all sstables are promoted for node in cluster.nodelist(): node.nodetool('compact ks tbl') self.assertAllRepairedSSTables(node, 'ks')
source = Cluster( ['192.168.52.1', '192.168.52.2'], port=9042) dest = Cluster( ['192.168.58.3', '192.168.58.4'], port=9042) FORMAT = "%(levelname)s %(asctime)-15s %(message)s" logging.basicConfig(format=FORMAT, level=logging.DEBUG, filename='kcopy.log') src_session = source.connect(keyspace=keyspace) dst_session = dest.connect(keyspace=keyspace) select = SimpleStatement('SELECT key, column1, value, TTL(value) FROM your_column_family', fetch_size=500) select.consistency_level = ConsistencyLevel.QUORUM insert = dst_session.prepare('INSERT INTO your_column_family (key, column1, value) VALUES (?, ?, ?)') insert_ttl = dst_session.prepare('INSERT INTO your_column_family (key, column1, value) VALUES (?, ?, ?) USING TTL ?') insert.consistency_level = ConsistencyLevel.QUORUM insert_ttl.consistency_level = ConsistencyLevel.QUORUM reading = True f = src_session.execute_async(select) w = None count = 0 def continue_read(): global reading if f.has_more_pages: