def compaction_throughput_test(self): """ Test setting compaction throughput. Set throughput, insert data and ensure compaction performance corresponds. """ cluster = self.cluster cluster.populate(1).start(wait_for_binary_proto=True) [node1] = cluster.nodelist() # disableautocompaction only disables compaction for existing tables, # so initialize stress tables with stress first stress_write(node1, keycount=1) node1.nodetool('disableautocompaction') stress_write(node1, keycount=200000) threshold = "5" node1.nodetool('setcompactionthroughput -- ' + threshold) matches = block_on_compaction_log(node1) stringline = matches[0] throughput_pattern = re.compile('''.* # it doesn't matter what the line starts with = # wait for an equals sign ([\s\d\.]*) # capture a decimal number, possibly surrounded by whitespace MB/s.* # followed by 'MB/s' ''', re.X) avgthroughput = re.match(throughput_pattern, stringline).group(1).strip() debug(avgthroughput) self.assertGreaterEqual(float(threshold), float(avgthroughput)) assert_almost_equal(float(threshold), float(avgthroughput), error=0.2)
def simple_bootstrap_test(self): cluster = self.cluster tokens = cluster.balanced_tokens(2) cluster.set_configuration_options(values={'num_tokens': 1}) debug("[node1, node2] tokens: %r" % (tokens, )) keys = 10000 # Create a single node cluster cluster.populate(1) node1 = cluster.nodelist()[0] node1.set_configuration_options(values={'initial_token': tokens[0]}) cluster.start(wait_other_notice=True) session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 1) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) # record the size before inserting any of our own data empty_size = node1.data_size() debug("node1 empty size : %s" % float(empty_size)) insert_statement = session.prepare( "INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')") execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)]) node1.flush() node1.compact() initial_size = node1.data_size() debug("node1 size before bootstrapping node2: %s" % float(initial_size)) # Reads inserted data all during the bootstrap process. We shouldn't # get any error reader = self.go(lambda _: query_c1c2( session, random.randint(0, keys - 1), ConsistencyLevel.ONE)) # Bootstrapping a new node node2 = new_node(cluster) node2.set_configuration_options(values={'initial_token': tokens[1]}) node2.start(wait_for_binary_proto=True) node2.compact() reader.check() node1.cleanup() debug("node1 size after cleanup: %s" % float(node1.data_size())) node1.compact() debug("node1 size after compacting: %s" % float(node1.data_size())) time.sleep(.5) reader.check() debug("node2 size after compacting: %s" % float(node2.data_size())) size1 = float(node1.data_size()) size2 = float(node2.data_size()) assert_almost_equal(size1, size2, error=0.3) assert_almost_equal(float(initial_size - empty_size), 2 * (size1 - float(empty_size)))
def compaction_throughput_test(self): """ Test setting compaction throughput. Set throughput, insert data and ensure compaction performance corresponds. """ cluster = self.cluster cluster.populate(1).start(wait_for_binary_proto=True) [node1] = cluster.nodelist() # disableautocompaction only disables compaction for existing tables, # so initialize stress tables with stress first stress_write(node1, keycount=1) node1.nodetool('disableautocompaction') stress_write(node1, keycount=200000) threshold = "5" node1.nodetool('setcompactionthroughput -- ' + threshold) matches = block_on_compaction_log(node1) stringline = matches[0] throughput_pattern = re.compile( '''.* # it doesn't matter what the line starts with = # wait for an equals sign ([\s\d\.]*) # capture a decimal number, possibly surrounded by whitespace MB/s.* # followed by 'MB/s' ''', re.X) avgthroughput = re.match(throughput_pattern, stringline).group(1).strip() debug(avgthroughput) self.assertGreaterEqual(float(threshold), float(avgthroughput)) assert_almost_equal(float(threshold), float(avgthroughput), error=0.2)
def multiple_subsequent_repair_test(self): """ Covers CASSANDRA-8366 There is an issue with subsequent inc repairs. """ cluster = self.cluster cluster.populate(3).start() [node1, node2, node3] = cluster.nodelist() debug("Inserting data with stress") node1.stress([ 'write', 'n=5M', '-rate', 'threads=10', '-schema', 'replication(factor=3)' ]) debug("Flushing nodes") cluster.flush() debug("Waiting compactions to finish") cluster.wait_for_compactions() if self.cluster.version() >= '2.2': debug("Repairing node1") node1.nodetool("repair") debug("Repairing node2") node2.nodetool("repair") debug("Repairing node3") node3.nodetool("repair") else: debug("Repairing node1") node1.nodetool("repair -par -inc") debug("Repairing node2") node2.nodetool("repair -par -inc") debug("Repairing node3") node3.nodetool("repair -par -inc") # Using "print" instead of debug() here is on purpose. The compactions # take a long time and don't print anything by default, which can result # in the test being timed out after 20 minutes. These print statements # prevent it from being timed out. print "compacting node1" node1.compact() print "compacting node2" node2.compact() print "compacting node3" node3.compact() # wait some time to be sure the load size is propagated between nodes debug("Waiting for load size info to be propagated between nodes") time.sleep(45) load_size_in_kb = float( sum(map(lambda n: n.data_size(), [node1, node2, node3]))) load_size = load_size_in_kb / 1024 / 1024 debug("Total Load size: {}GB".format(load_size)) # There is still some overhead, but it's lot better. We tolerate 25%. expected_load_size = 4.5 # In GB assert_almost_equal(load_size, expected_load_size, error=0.25)
def ttl_is_replicated_test(self): """ Test that the ttl setting is replicated properly on all nodes """ self.prepare(default_time_to_live=5) cursor1 = self.patient_exclusive_cql_connection(self.node1) cursor2 = self.patient_exclusive_cql_connection(self.node2) cursor1.execute("USE ks;") cursor2.execute("USE ks;") query = SimpleStatement( "INSERT INTO ttl_table (key, col1) VALUES (1, 1);", consistency_level=ConsistencyLevel.ALL ) cursor1.execute(query) assert_all( cursor1, "SELECT * FROM ttl_table;", [[1, 1, None, None]], cl=ConsistencyLevel.ALL ) ttl_cursor1 = cursor1.execute('SELECT ttl(col1) FROM ttl_table;') ttl_cursor2 = cursor2.execute('SELECT ttl(col1) FROM ttl_table;') assert_almost_equal(ttl_cursor1[0][0], ttl_cursor2[0][0], error=0.05) time.sleep(7) assert_none(cursor1, "SELECT * FROM ttl_table;", cl=ConsistencyLevel.ALL)
def assert_balanced(self, node): sums = [] for sstabledir in node.get_sstables_per_data_directory('keyspace1', 'standard1'): sum = 0 for sstable in sstabledir: sum = sum + os.path.getsize(sstable) sums.append(sum) assert_almost_equal(*sums, error=0.2, error_message=node.name)
def multiple_subsequent_repair_test(self): """ Covers CASSANDRA-8366 There is an issue with subsequent inc repairs. """ cluster = self.cluster cluster.set_configuration_options(values={ 'compaction_throughput_mb_per_sec': 0 }) cluster.populate(3).start() [node1,node2,node3] = cluster.nodelist() debug("Inserting data with stress") expected_load_size = 4.5 # In GB node1.stress(['write', 'n=5M', '-rate', 'threads=50', '-schema', 'replication(factor=3)']) debug("Flushing nodes") node1.flush() node2.flush() node3.flush() if self.cluster.version() >= '3.0': debug("Repairing node1") node1.nodetool("repair") debug("Repairing node2") node2.nodetool("repair") debug("Repairing node3") node3.nodetool("repair") else: debug("Repairing node1") node1.nodetool("repair -par -inc") debug("Repairing node2") node2.nodetool("repair -par -inc") debug("Repairing node3") node3.nodetool("repair -par -inc") # Using "print" instead of debug() here is on purpose. The compactions # take a long time and don't print anything by default, which can result # in the test being timed out after 20 minutes. These print statements # prevent it from being timed out. print "compacting node1" node1.compact() print "compacting node2" node2.compact() print "compacting node3" node3.compact() # wait some time to be sure the load size is propagated between nodes debug("Waiting for load size info to be propagated between nodes") time.sleep(45) load_size_in_kb = float( sum(map(lambda n: n.data_size(), [node1, node2, node3])) ) load_size = load_size_in_kb/1024/1024 debug("Total Load size: {}GB".format(load_size)) # There is still some overhead, but it's lot better. We tolerate 25%. assert_almost_equal(load_size, expected_load_size, error=0.25)
def assertions_test(self): # assert_exception_test mock_session = Mock( ** {'execute.side_effect': AlreadyExists("Dummy exception message.")}) assert_exception(mock_session, "DUMMY QUERY", expected=AlreadyExists) # assert_unavailable_test mock_session = Mock(**{ 'execute.side_effect': Unavailable("Dummy Unavailabile message.") }) assert_unavailable(mock_session.execute) # assert_invalid_test mock_session = Mock(**{ 'execute.side_effect': InvalidRequest("Dummy InvalidRequest message.") }) assert_invalid(mock_session, "DUMMY QUERY") # assert_unauthorized_test mock_session = Mock(**{ 'execute.side_effect': Unauthorized("Dummy Unauthorized message.") }) assert_unauthorized(mock_session, "DUMMY QUERY", None) # assert_one_test mock_session = Mock() mock_session.execute = Mock(return_value=[[1, 1]]) assert_one(mock_session, "SELECT * FROM test", [1, 1]) # assert_none_test mock_session = Mock() mock_session.execute = Mock(return_value=[]) assert_none(mock_session, "SELECT * FROM test") # assert_all_test mock_session = Mock() mock_session.execute = Mock(return_value=[[i, i] for i in range(0, 10)]) assert_all(mock_session, "SELECT k, v FROM test", [[i, i] for i in range(0, 10)], ignore_order=True) # assert_almost_equal_test assert_almost_equal(1, 1.1, 1.2, 1.9, error=1.0) # assert_row_count_test mock_session = Mock() mock_session.execute = Mock(return_value=[[1]]) assert_row_count(mock_session, 'test', 1) # assert_length_equal_test check = [1, 2, 3, 4] assert_length_equal(check, 4)
def simple_bootstrap_test(self): cluster = self.cluster tokens = cluster.balanced_tokens(2) cluster.set_configuration_options(values={'num_tokens': 1}) debug("[node1, node2] tokens: %r" % (tokens,)) keys = 10000 # Create a single node cluster cluster.populate(1) node1 = cluster.nodelist()[0] node1.set_configuration_options(values={'initial_token': tokens[0]}) cluster.start(wait_other_notice=True) session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 1) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) # record the size before inserting any of our own data empty_size = node1.data_size() debug("node1 empty size : %s" % float(empty_size)) insert_statement = session.prepare("INSERT INTO ks.cf (key, c1, c2) VALUES (?, 'value1', 'value2')") execute_concurrent_with_args(session, insert_statement, [['k%d' % k] for k in range(keys)]) node1.flush() node1.compact() initial_size = node1.data_size() debug("node1 size before bootstrapping node2: %s" % float(initial_size)) # Reads inserted data all during the bootstrap process. We shouldn't # get any error reader = self.go(lambda _: query_c1c2(session, random.randint(0, keys - 1), ConsistencyLevel.ONE)) # Bootstrapping a new node node2 = new_node(cluster) node2.set_configuration_options(values={'initial_token': tokens[1]}) node2.start(wait_for_binary_proto=True) node2.compact() reader.check() node1.cleanup() debug("node1 size after cleanup: %s" % float(node1.data_size())) node1.compact() debug("node1 size after compacting: %s" % float(node1.data_size())) time.sleep(.5) reader.check() debug("node2 size after compacting: %s" % float(node2.data_size())) size1 = float(node1.data_size()) size2 = float(node2.data_size()) assert_almost_equal(size1, size2, error=0.3) assert_almost_equal(float(initial_size - empty_size), 2 * (size1 - float(empty_size)))
def decommission_test(self): cluster = self.cluster tokens = cluster.balanced_tokens(4) cluster.populate(4, tokens=tokens).start() node1, node2, node3, node4 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 2) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=30000, consistency=ConsistencyLevel.QUORUM) cluster.flush() sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()] init_size = sizes[0] assert_almost_equal(*sizes) time.sleep(.5) node4.decommission() node4.stop() cluster.cleanup() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 30000): query_c1c2(session, n, ConsistencyLevel.QUORUM) sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()] debug(sizes) assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal((2.0 / 3.0) * sizes[0], sizes[2]) assert_almost_equal(sizes[2], init_size)
def ttl_is_respected_on_repair_test(self): """ Test that ttl is respected on repair """ self.prepare() self.session1.execute(""" ALTER KEYSPACE ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}; """) self.session1.execute(""" INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5; """) self.session1.execute(""" INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 1000; """) assert_all( self.session1, "SELECT * FROM ttl_table;", [[1, 1, None, None], [2, 2, None, None]] ) time.sleep(7) self.node1.stop() session2 = self.patient_exclusive_cql_connection(self.node2) session2.execute("USE ks;") assert_unavailable(session2.execute, "SELECT * FROM ttl_table;") self.node1.start(wait_for_binary_proto=True) self.session1 = self.patient_exclusive_cql_connection(self.node1) self.session1.execute("USE ks;") self.session1.execute(""" ALTER KEYSPACE ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 2}; """) self.node1.repair(['ks']) ttl_start = time.time() ttl_session1 = self.session1.execute('SELECT ttl(col1) FROM ttl_table;') self.node1.stop() assert_row_count(session2, 'ttl_table', 1) assert_all( session2, "SELECT * FROM ttl_table;", [[2, 2, None, None]] ) # Check that the TTL on both server are the same ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;') ttl_session1 = ttl_session1[0][0] - (time.time() - ttl_start) assert_almost_equal(ttl_session1, ttl_session2[0][0], error=0.005)
def _segment_size_test(self, segment_size_in_mb, compressed=False): """ Execute a basic commitlog test and validate the commitlog files """ conf = {'commitlog_segment_size_in_mb': segment_size_in_mb} if compressed: conf['commitlog_compression'] = [{'class_name': 'LZ4Compressor'}] conf['memtable_heap_space_in_mb'] = 512 self.prepare(configuration=conf, create_test_keyspace=False) segment_size = segment_size_in_mb * 1024 * 1024 self.node1.stress( ['write', 'n=150k', 'no-warmup', '-rate', 'threads=25']) time.sleep(1) commitlogs = self._get_commitlog_files() self.assertGreater(len(commitlogs), 0, 'No commit log files were created') # the most recently-written segment of the commitlog may be smaller # than the expected size, so we allow exactly one segment to be smaller smaller_found = False for i, f in enumerate(commitlogs): size = os.path.getsize(f) size_in_mb = int(size / 1024 / 1024) debug('segment file {} {}; smaller already found: {}'.format( f, size_in_mb, smaller_found)) if size_in_mb < 1 or size < (segment_size * 0.1): debug('segment file not yet used; moving to next file') continue # commitlog not yet used try: if compressed: # if compression is used, we assume there will be at most a 50% compression ratio self.assertLess(size, segment_size) self.assertGreater(size, segment_size / 2) else: # if no compression is used, the size will be close to what we expect assert_almost_equal(size, segment_size, error=0.05) except AssertionError as e: # the last segment may be smaller if not smaller_found: self.assertLessEqual(size, segment_size) smaller_found = True else: raise e
def assertions_test(self): # assert_exception_test mock_session = Mock(**{'execute.side_effect': AlreadyExists("Dummy exception message.")}) assert_exception(mock_session, "DUMMY QUERY", expected=AlreadyExists) # assert_unavailable_test mock_session = Mock(**{'execute.side_effect': Unavailable("Dummy Unavailabile message.")}) assert_unavailable(mock_session.execute) # assert_invalid_test mock_session = Mock(**{'execute.side_effect': InvalidRequest("Dummy InvalidRequest message.")}) assert_invalid(mock_session, "DUMMY QUERY") # assert_unauthorized_test mock_session = Mock(**{'execute.side_effect': Unauthorized("Dummy Unauthorized message.")}) assert_unauthorized(mock_session, "DUMMY QUERY", None) # assert_one_test mock_session = Mock() mock_session.execute = Mock(return_value=[[1, 1]]) assert_one(mock_session, "SELECT * FROM test", [1, 1]) # assert_none_test mock_session = Mock() mock_session.execute = Mock(return_value=[]) assert_none(mock_session, "SELECT * FROM test") # assert_all_test mock_session = Mock() mock_session.execute = Mock(return_value=[[i, i] for i in range(0, 10)]) assert_all(mock_session, "SELECT k, v FROM test", [[i, i] for i in range(0, 10)], ignore_order=True) # assert_almost_equal_test assert_almost_equal(1, 1.1, 1.2, 1.9, error=1.0) # assert_row_count_test mock_session = Mock() mock_session.execute = Mock(return_value=[[1]]) assert_row_count(mock_session, 'test', 1) # assert_length_equal_test check = [1, 2, 3, 4] assert_length_equal(check, 4)
def _segment_size_test(self, segment_size_in_mb, compressed=False): """ Execute a basic commitlog test and validate the commitlog files """ conf = {'commitlog_segment_size_in_mb': segment_size_in_mb} if compressed: conf['commitlog_compression'] = [{'class_name': 'LZ4Compressor'}] conf['memtable_heap_space_in_mb'] = 512 self.prepare(configuration=conf, create_test_keyspace=False) segment_size = segment_size_in_mb * 1024 * 1024 self.node1.stress(['write', 'n=150k', '-rate', 'threads=25']) time.sleep(1) commitlogs = self._get_commitlog_files() self.assertGreater(len(commitlogs), 0, 'No commit log files were created') # the most recently-written segment of the commitlog may be smaller # than the expected size, so we allow exactly one segment to be smaller smaller_found = False for i, f in enumerate(commitlogs): size = os.path.getsize(f) size_in_mb = int(size / 1024 / 1024) debug('segment file {} {}; smaller already found: {}'.format(f, size_in_mb, smaller_found)) if size_in_mb < 1 or size < (segment_size * 0.1): debug('segment file not yet used; moving to next file') continue # commitlog not yet used try: if compressed: # if compression is used, we assume there will be at most a 50% compression ratio self.assertLess(size, segment_size) self.assertGreater(size, segment_size / 2) else: # if no compression is used, the size will be close to what we expect assert_almost_equal(size, segment_size, error=0.05) except AssertionError as e: # the last segment may be smaller if not smaller_found: self.assertLessEqual(size, segment_size) smaller_found = True else: raise e
def ttl_is_respected_on_repair_test(self): """ Test that ttl is respected on repair """ self.prepare() self.session1.execute(""" ALTER KEYSPACE ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 1}; """) self.session1.execute(""" INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5; """) self.session1.execute(""" INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 1000; """) assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 1, None, None], [2, 2, None, None]]) time.sleep(7) self.node1.stop() session2 = self.patient_exclusive_cql_connection(self.node2) session2.execute("USE ks;") assert_unavailable(session2.execute, "SELECT * FROM ttl_table;") self.node1.start(wait_for_binary_proto=True) self.session1 = self.patient_exclusive_cql_connection(self.node1) self.session1.execute("USE ks;") self.session1.execute(""" ALTER KEYSPACE ks WITH REPLICATION = {'class' : 'SimpleStrategy', 'replication_factor' : 2}; """) self.node1.repair(['ks']) ttl_start = time.time() ttl_session1 = self.session1.execute( 'SELECT ttl(col1) FROM ttl_table;') self.node1.stop() assert_row_count(session2, 'ttl_table', 1) assert_all(session2, "SELECT * FROM ttl_table;", [[2, 2, None, None]]) # Check that the TTL on both server are the same ttl_session2 = session2.execute('SELECT ttl(col1) FROM ttl_table;') ttl_session1 = ttl_session1[0][0] - (time.time() - ttl_start) assert_almost_equal(ttl_session1, ttl_session2[0][0], error=0.005)
def _commitlog_test(self, segment_size_in_mb, commitlog_size, num_commitlog_files, compressed=False, files_error=0): """ Execute a basic commitlog test and validate the commitlog files """ if compressed: segment_size_in_mb *= 0.7 segment_size = segment_size_in_mb * 1024 * 1024 self.node1.stress(['write', 'n=150000', '-rate', 'threads=25']) time.sleep(1) if not ccmlib.common.is_win(): tolerated_error = 0.15 if compressed else 0.05 assert_almost_equal(self._get_commitlog_size(), commitlog_size, error=tolerated_error) commitlogs = self._get_commitlog_files() assert_almost_equal(len(commitlogs), num_commitlog_files, error=files_error) for f in commitlogs: size = os.path.getsize(f) size_in_mb = int(size/1024/1024) if size_in_mb < 1 or size < (segment_size*0.1): continue # commitlog not yet used tolerated_error = 0.15 if compressed else 0.05 assert_almost_equal(size, segment_size, error=tolerated_error)
def ttl_is_respected_on_delayed_replication_test(self): """ Test that ttl is respected on delayed replication """ self.prepare() self.node2.stop() self.cursor1.execute(""" INSERT INTO ttl_table (key, col1) VALUES (1, 1) USING TTL 5; """) self.cursor1.execute(""" INSERT INTO ttl_table (key, col1) VALUES (2, 2) USING TTL 60; """) assert_all( self.cursor1, "SELECT * FROM ttl_table;", [[1, 1, None, None], [2, 2, None, None]] ) time.sleep(7) self.node1.stop() self.node2.start(wait_for_binary_proto=True) cursor2 = self.patient_exclusive_cql_connection(self.node2) cursor2.execute("USE ks;") assert_row_count(cursor2, 'ttl_table', 0) # should be 0 since node1 is down, no replica yet self.node1.start(wait_for_binary_proto=True) self.cursor1 = self.patient_exclusive_cql_connection(self.node1) self.cursor1.execute("USE ks;") self.node1.cleanup() # Check that the expired data has not been replicated assert_row_count(cursor2, 'ttl_table', 1) assert_all( cursor2, "SELECT * FROM ttl_table;", [[2, 2, None, None]], cl=ConsistencyLevel.ALL ) # Check that the TTL on both server are the same ttl_cursor1 = self.cursor1.execute('SELECT ttl(col1) FROM ttl_table;') ttl_cursor2 = cursor2.execute('SELECT ttl(col1) FROM ttl_table;') assert_almost_equal(ttl_cursor1[0][0], ttl_cursor2[0][0], error=0.1)
def movement_test(self): cluster = self.cluster # Create an unbalanced ring cluster.populate(3, tokens=[0, 2**48, 2**62]).start() node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 1) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE) cluster.flush() # Move nodes to balance the cluster balancing_tokens = cluster.balanced_tokens(3) escformat = '%s' node1.move(escformat % balancing_tokens[0]) # can't assume 0 is balanced with m3p node2.move(escformat % balancing_tokens[1]) node3.move(escformat % balancing_tokens[2]) time.sleep(1) cluster.cleanup() # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(session, n, ConsistencyLevel.ONE) # Now the load should be basically even sizes = [node.data_size() for node in [node1, node2, node3]] assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal(sizes[0], sizes[2]) assert_almost_equal(sizes[1], sizes[2])
def decommission_test(self): cluster = self.cluster tokens = cluster.balanced_tokens(4) cluster.populate(4, tokens=tokens).start() node1, node2, node3, node4 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 2) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=30000, consistency=ConsistencyLevel.QUORUM) cluster.flush() sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ] init_size = sizes[0] assert_almost_equal(*sizes) time.sleep(.5) node4.decommission() node4.stop() cluster.cleanup() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 30000): query_c1c2(session, n, ConsistencyLevel.QUORUM) sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ] debug(sizes) assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal((2.0 / 3.0) * sizes[0], sizes[2]) assert_almost_equal(sizes[2], init_size)
def _commitlog_test(self, segment_size_in_mb, commitlog_size, num_commitlog_files, compressed=False, files_error=0): """ Execute a basic commitlog test and validate the commitlog files """ if compressed: segment_size_in_mb *= 0.7 segment_size = segment_size_in_mb * 1024 * 1024 self.node1.stress(['write', 'n=150000', '-rate', 'threads=25']) time.sleep(1) commitlogs = self._get_commitlog_files() assert_almost_equal(len(commitlogs), num_commitlog_files, error=files_error) if not ccmlib.common.is_win(): tolerated_error = 0.15 if compressed else 0.05 assert_almost_equal(sum([int(os.path.getsize(f)/1024/1024) for f in commitlogs]), commitlog_size, error=tolerated_error) # the most recently-written segment of the commitlog may be smaller # than the expected size, so we allow exactly one segment to be smaller smaller_found = False for i, f in enumerate(commitlogs): size = os.path.getsize(f) size_in_mb = int(size/1024/1024) debug('segment file {} {}; smaller already found: {}'.format(f, size_in_mb, smaller_found)) if size_in_mb < 1 or size < (segment_size*0.1): continue # commitlog not yet used tolerated_error = 0.15 if compressed else 0.05 try: # in general, the size will be close to what we expect assert_almost_equal(size, segment_size, error=tolerated_error) except AssertionError as e: # but segments may be smaller with compression enabled, # or the last segment may be smaller if (not smaller_found) or compressed: self.assertLessEqual(size, segment_size) smaller_found = True else: raise e
def movement_test(self): cluster = self.cluster # Create an unbalanced ring cluster.populate(3, tokens=[0, 2**48, 2**62]).start() node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 1) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=30000, consistency=ConsistencyLevel.ONE) cluster.flush() # Move nodes to balance the cluster def move_node(node, token, ip): mark = node.mark_log() node.move(token) # can't assume 0 is balanced with m3p node.watch_log_for('{} state jump to NORMAL'.format(ip), from_mark=mark, timeout=180) time.sleep(3) balancing_tokens = cluster.balanced_tokens(3) move_node(node1, balancing_tokens[0], '127.0.0.1') move_node(node2, balancing_tokens[1], '127.0.0.2') move_node(node3, balancing_tokens[2], '127.0.0.3') time.sleep(1) cluster.cleanup() # Check we can get all the keys for n in xrange(0, 30000): query_c1c2(session, n, ConsistencyLevel.ONE) # Now the load should be basically even sizes = [node.data_size() for node in [node1, node2, node3]] assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal(sizes[0], sizes[2]) assert_almost_equal(sizes[1], sizes[2])
def movement_test(self): cluster = self.cluster # Create an unbalanced ring cluster.populate(3, tokens=[0, 2**48, 2**62]).start() node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 1) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE) cluster.flush() # Move nodes to balance the cluster def move_node(node, token, ip): mark = node.mark_log() node.move(token) # can't assume 0 is balanced with m3p node.watch_log_for('{} state jump to NORMAL'.format(ip), from_mark=mark, timeout=180) time.sleep(3) balancing_tokens = cluster.balanced_tokens(3) move_node(node1, balancing_tokens[0], '127.0.0.1') move_node(node2, balancing_tokens[1], '127.0.0.2') move_node(node3, balancing_tokens[2], '127.0.0.3') time.sleep(1) cluster.cleanup() # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(session, n, ConsistencyLevel.ONE) # Now the load should be basically even sizes = [node.data_size() for node in [node1, node2, node3]] assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal(sizes[0], sizes[2]) assert_almost_equal(sizes[1], sizes[2])
def movement_test(self): cluster = self.cluster # Create an unbalanced ring cluster.populate(3, tokens=[0, 2**48, 2**62]).start() node1, node2, node3 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 1) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=10000, consistency=ConsistencyLevel.ONE) cluster.flush() # Move nodes to balance the cluster balancing_tokens = cluster.balanced_tokens(3) escformat = '\\%s' if cluster.version() >= '2.1': escformat = '%s' node1.move(escformat % balancing_tokens[0]) # can't assume 0 is balanced with m3p node2.move(escformat % balancing_tokens[1]) node3.move(escformat % balancing_tokens[2]) time.sleep(1) cluster.cleanup() # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(session, n, ConsistencyLevel.ONE) # Now the load should be basically even sizes = [node.data_size() for node in [node1, node2, node3]] assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal(sizes[0], sizes[2]) assert_almost_equal(sizes[1], sizes[2])
def decomission_test(self): cluster = self.cluster tokens = cluster.balanced_tokens(4) cluster.populate(4, tokens=tokens).start() node1, node2, node3, node4 = cluster.nodelist() cursor = self.patient_cql_connection(node1) self.create_ks(cursor, 'ks', 2) self.create_cf(cursor, 'cf',columns={'c1': 'text', 'c2': 'text'}) for n in xrange(0, 10000): insert_c1c2(cursor, n, ConsistencyLevel.QUORUM) cluster.flush() sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running()] init_size = sizes[0] assert_almost_equal(*sizes) time.sleep(.5) node4.decommission() node4.stop() cluster.cleanup() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(cursor, n, ConsistencyLevel.QUORUM) sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ] three_node_sizes = sizes assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal((2.0/3.0) * sizes[0], sizes[2]) assert_almost_equal(sizes[2], init_size) if cluster.version() <= '1.2': node3.stop(wait_other_notice=True) node1.removeToken(tokens[2]) time.sleep(.5) cluster.cleanup() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(cursor, n, ConsistencyLevel.QUORUM) sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ] assert_almost_equal(*sizes) assert_almost_equal(sizes[0], 2 * init_size) node5 = new_node(cluster, token=(tokens[2]+1)).start() time.sleep(.5) cluster.cleanup() time.sleep(.5) cluster.compact() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(cursor, n, ConsistencyLevel.QUORUM) sizes = [ node.data_size() for node in cluster.nodelist() if node.is_running() ] # We should be back to the earlir 3 nodes situation for i in xrange(0, len(sizes)): assert_almost_equal(sizes[i], three_node_sizes[i])
def decommission_test(self): cluster = self.cluster tokens = cluster.balanced_tokens(4) cluster.populate(4, tokens=tokens).start() node1, node2, node3, node4 = cluster.nodelist() session = self.patient_cql_connection(node1) self.create_ks(session, 'ks', 2) self.create_cf(session, 'cf', columns={'c1': 'text', 'c2': 'text'}) insert_c1c2(session, n=10000, consistency=ConsistencyLevel.QUORUM) cluster.flush() sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()] init_size = sizes[0] assert_almost_equal(*sizes) time.sleep(.5) node4.decommission() node4.stop() cluster.cleanup() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(session, n, ConsistencyLevel.QUORUM) sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()] three_node_sizes = sizes assert_almost_equal(sizes[0], sizes[1]) assert_almost_equal((2.0 / 3.0) * sizes[0], sizes[2]) assert_almost_equal(sizes[2], init_size) if cluster.version() <= '1.2': node3.stop(wait_other_notice=True) node1.removeToken(tokens[2]) time.sleep(.5) cluster.cleanup() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(session, n, ConsistencyLevel.QUORUM) sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()] assert_almost_equal(*sizes) assert_almost_equal(sizes[0], 2 * init_size) node5 = new_node(cluster, token=(tokens[2] + 1)).start() time.sleep(.5) cluster.cleanup() time.sleep(.5) cluster.compact() time.sleep(.5) # Check we can get all the keys for n in xrange(0, 10000): query_c1c2(session, n, ConsistencyLevel.QUORUM) sizes = [node.data_size() for node in cluster.nodelist() if node.is_running()] # We should be back to the earlir 3 nodes situation for i in xrange(0, len(sizes)): assert_almost_equal(sizes[i], three_node_sizes[i])
def multiple_subsequent_repair_test(self): """ @jira_ticket CASSANDRA-8366 There is an issue with subsequent inc repairs increasing load size. So we perform several repairs and check that the expected amount of data exists. * Launch a three node cluster * Write 5M rows with stress * Wait for minor compactions to finish * Issue an incremental repair on each node, sequentially * Issue major compactions on each node * Sleep for a while so load size can be propagated between nodes * Verify the correct amount of data is on each node """ cluster = self.cluster cluster.populate(3).start() node1, node2, node3 = cluster.nodelist() debug("Inserting data with stress") node1.stress(['write', 'n=5M', '-rate', 'threads=10', '-schema', 'replication(factor=3)']) debug("Flushing nodes") cluster.flush() debug("Waiting compactions to finish") cluster.wait_for_compactions() if self.cluster.version() >= '2.2': debug("Repairing node1") node1.nodetool("repair") debug("Repairing node2") node2.nodetool("repair") debug("Repairing node3") node3.nodetool("repair") else: debug("Repairing node1") node1.nodetool("repair -par -inc") debug("Repairing node2") node2.nodetool("repair -par -inc") debug("Repairing node3") node3.nodetool("repair -par -inc") # Using "print" instead of debug() here is on purpose. The compactions # take a long time and don't print anything by default, which can result # in the test being timed out after 20 minutes. These print statements # prevent it from being timed out. print "compacting node1" node1.compact() print "compacting node2" node2.compact() print "compacting node3" node3.compact() # wait some time to be sure the load size is propagated between nodes debug("Waiting for load size info to be propagated between nodes") time.sleep(45) load_size_in_kb = float(sum(map(lambda n: n.data_size(), [node1, node2, node3]))) load_size = load_size_in_kb / 1024 / 1024 debug("Total Load size: {}GB".format(load_size)) # There is still some overhead, but it's lot better. We tolerate 25%. expected_load_size = 4.5 # In GB assert_almost_equal(load_size, expected_load_size, error=0.25)