def _token_gen_test(self, nodes, randomPart=None):
generated_tokens, session = self.prepare(randomPart, nodes=nodes)
dc_tokens = generated_tokens[0]
tokens = []
local_tokens = rows_to_list(
session.execute("SELECT tokens FROM system.local"))[0]
self.assertEqual(local_tokens.__len__(), 1, "too many tokens for peer")
for tok in local_tokens:
tokens += tok
rows = rows_to_list(session.execute("SELECT tokens FROM system.peers"))
self.assertEqual(rows.__len__(), nodes - 1)
for row in rows:
peer_tokens = row[0]
self.assertEqual(peer_tokens.__len__(), 1,
"too many tokens for peer")
for tok in peer_tokens:
tokens.append(tok)
self.assertEqual(tokens.__len__(), dc_tokens.__len__())
for cluster_token in tokens:
tok = int(cluster_token)
self.assertGreaterEqual(
dc_tokens.index(tok), 0,
"token in cluster does not match generated tokens")
def test_query_indexes_with_vnodes(self):
"""
Verifies correct query behaviour in the presence of vnodes
@jira_ticket CASSANDRA-11104
"""
cluster = self.cluster
cluster.populate(2).start()
node1, node2 = cluster.nodelist()
session = self.patient_cql_connection(node1)
session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': '1'};")
session.execute("CREATE TABLE ks.compact_table (a int PRIMARY KEY, b int) WITH COMPACT STORAGE;")
session.execute("CREATE INDEX keys_index ON ks.compact_table (b);")
session.execute("CREATE TABLE ks.regular_table (a int PRIMARY KEY, b int)")
session.execute("CREATE INDEX composites_index on ks.regular_table (b)")
insert_args = [(i, i % 2) for i in xrange(100)]
execute_concurrent_with_args(session,
session.prepare("INSERT INTO ks.compact_table (a, b) VALUES (?, ?)"),
insert_args)
execute_concurrent_with_args(session,
session.prepare("INSERT INTO ks.regular_table (a, b) VALUES (?, ?)"),
insert_args)
res = session.execute("SELECT * FROM ks.compact_table WHERE b = 0")
self.assertEqual(len(rows_to_list(res)), 50)
res = session.execute("SELECT * FROM ks.regular_table WHERE b = 0")
self.assertEqual(len(rows_to_list(res)), 50)
def table_test(self):
"""
CREATE TABLE, ALTER TABLE, TRUNCATE TABLE, DROP TABLE statements
"""
session = self.prepare()
session.execute("CREATE TABLE test1 (k int PRIMARY KEY, v1 int)")
session.execute("CREATE TABLE test2 (k int, c1 int, v1 int, PRIMARY KEY (k, c1)) WITH COMPACT STORAGE")
session.execute("ALTER TABLE test1 ADD v2 int")
for i in xrange(0, 10):
session.execute("INSERT INTO test1 (k, v1, v2) VALUES (%d, %d, %d)" % (i, i, i))
session.execute("INSERT INTO test2 (k, c1, v1) VALUES (%d, %d, %d)" % (i, i, i))
res = sorted(session.execute("SELECT * FROM test1"))
assert rows_to_list(res) == [[i, i, i] for i in xrange(0, 10)], res
res = sorted(session.execute("SELECT * FROM test2"))
assert rows_to_list(res) == [[i, i, i] for i in xrange(0, 10)], res
session.execute("TRUNCATE test1")
session.execute("TRUNCATE test2")
res = session.execute("SELECT * FROM test1")
assert rows_to_list(res) == [], res
res = session.execute("SELECT * FROM test2")
assert rows_to_list(res) == [], res
session.execute("DROP TABLE test1")
session.execute("DROP TABLE test2")
assert_invalid(session, "SELECT * FROM test1", expected=InvalidRequest)
assert_invalid(session, "SELECT * FROM test2", expected=InvalidRequest)
def drop_column_and_restart_test(self):
"""
Simply insert data in a table, drop a column involved in the insert and restart the node afterwards.
This ensures that the dropped_columns system table is properly flushed on the alter or the restart
fails as in CASSANDRA-11050.
@jira_ticket CASSANDRA-11050
"""
session = self.prepare()
session.execute("USE ks")
session.execute("CREATE TABLE t (k int PRIMARY KEY, c1 int, c2 int)")
session.execute("INSERT INTO t (k, c1, c2) VALUES (0, 0, 0)")
session.execute("ALTER TABLE t DROP c2")
rows = session.execute("SELECT * FROM t")
self.assertEqual([[0, 0]], rows_to_list(rows))
self.cluster.stop()
self.cluster.start()
session = self.patient_cql_connection(self.cluster.nodelist()[0])
session.execute("USE ks")
rows = session.execute("SELECT * FROM t")
self.assertEqual([[0, 0]], rows_to_list(rows))
def _test_bulk_round_trip(self, nodes, partitioner, num_records):
"""
Test exporting a large number of rows into a csv file.
"""
self.prepare(nodes=nodes, partitioner=partitioner)
self.node1.stress(['write', 'n={}'.format(num_records), '-rate', 'threads=50'])
stress_table = 'keyspace1.standard1'
self.assertEqual([[num_records]], rows_to_list(self.session.execute("SELECT COUNT(*) FROM {}".format(stress_table))))
self.tempfile = NamedTemporaryFile(delete=False)
debug('Exporting to csv file: {}'.format(self.tempfile.name))
start = datetime.datetime.now()
self.node1.run_cqlsh(cmds="COPY {} TO '{}'".format(stress_table, self.tempfile.name))
debug("COPY TO took {} to export {} records".format(datetime.datetime.now() - start, num_records))
# check all records were exported
self.assertEqual(num_records, sum(1 for line in open(self.tempfile.name)))
self.session.execute("TRUNCATE {}".format(stress_table))
debug('Importing from csv file: {}'.format(self.tempfile.name))
start = datetime.datetime.now()
self.node1.run_cqlsh(cmds="COPY {} FROM '{}'".format(stress_table, self.tempfile.name))
debug("COPY FROM took {} to import {} records".format(datetime.datetime.now() - start, num_records))
self.assertEqual([[num_records]], rows_to_list(self.session.execute("SELECT COUNT(*) FROM {}".format(stress_table))))
def drop_column_and_restart_test(self):
"""
Simply insert data in a table, drop a column involved in the insert and restart the node afterwards.
This ensures that the dropped_columns system table is properly flushed on the alter or the restart
fails as in CASSANDRA-11050.
@jira_ticket CASSANDRA-11050
"""
session = self.prepare()
session.execute("USE ks")
session.execute("CREATE TABLE t (k int PRIMARY KEY, c1 int, c2 int)")
session.execute("INSERT INTO t (k, c1, c2) VALUES (0, 0, 0)")
session.execute("ALTER TABLE t DROP c2")
rows = session.execute("SELECT * FROM t")
self.assertEqual([[0, 0]], rows_to_list(rows))
self.cluster.stop()
self.cluster.start()
session = self.patient_cql_connection(self.cluster.nodelist()[0])
session.execute("USE ks")
rows = session.execute("SELECT * FROM t")
self.assertEqual([[0, 0]], rows_to_list(rows))
def table_test(self):
"""
CREATE TABLE, ALTER TABLE, TRUNCATE TABLE, DROP TABLE statements
"""
session = self.prepare()
session.execute("CREATE TABLE test1 (k int PRIMARY KEY, v1 int)")
session.execute("CREATE TABLE test2 (k int, c1 int, v1 int, PRIMARY KEY (k, c1)) WITH COMPACT STORAGE")
session.execute("ALTER TABLE test1 ADD v2 int")
for i in xrange(0, 10):
session.execute("INSERT INTO test1 (k, v1, v2) VALUES (%d, %d, %d)" % (i, i, i))
session.execute("INSERT INTO test2 (k, c1, v1) VALUES (%d, %d, %d)" % (i, i, i))
res = sorted(session.execute("SELECT * FROM test1"))
assert rows_to_list(res) == [[i, i, i] for i in xrange(0, 10)], res
res = sorted(session.execute("SELECT * FROM test2"))
assert rows_to_list(res) == [[i, i, i] for i in xrange(0, 10)], res
session.execute("TRUNCATE test1")
session.execute("TRUNCATE test2")
res = session.execute("SELECT * FROM test1")
assert rows_to_list(res) == [], res
res = session.execute("SELECT * FROM test2")
assert rows_to_list(res) == [], res
session.execute("DROP TABLE test1")
session.execute("DROP TABLE test2")
assert_invalid(session, "SELECT * FROM test1", expected=InvalidRequest)
assert_invalid(session, "SELECT * FROM test2", expected=InvalidRequest)
def query_user(self, session, userid, age, consistency, check_ret=True):
statement = SimpleStatement("SELECT userid, age FROM users where userid = {}".format(userid), consistency_level=consistency)
res = session.execute(statement)
expected = [[userid, age]] if age else []
ret = rows_to_list(res) == expected
if check_ret:
self.assertTrue(ret, "Got {} from {}, expected {} at {}".format(rows_to_list(res), session.cluster.contact_points, expected, consistency_value_to_name(consistency)))
return ret
def query_user(self, session, userid, age, consistency, check_ret=True):
statement = SimpleStatement("SELECT userid, age FROM users where userid = %d" % (userid,), consistency_level=consistency)
res = session.execute(statement)
expected = [[userid, age]] if age else []
ret = rows_to_list(res) == expected
if check_ret:
assert ret, "Got %s from %s, expected %s at %s" % (rows_to_list(res), session.cluster.contact_points, expected, self._name(consistency))
return ret
def query_user(self, session, userid, age, consistency, check_ret=True):
statement = SimpleStatement("SELECT userid, age FROM users where userid = {}".format(userid), consistency_level=consistency)
res = session.execute(statement)
expected = [[userid, age]] if age else []
ret = rows_to_list(res) == expected
if check_ret:
self.assertTrue(ret, "Got {} from {}, expected {} at {}".format(rows_to_list(res), session.cluster.contact_points, expected, consistency_value_to_name(consistency)))
return ret
def test_commitlog_replay_on_startup(self):
"""
Test commit log replay
"""
node1 = self.node1
node1.set_configuration_options(batch_commitlog=True)
node1.start()
debug("Insert data")
session = self.patient_cql_connection(node1)
self.create_ks(session, 'Test', 1)
session.execute("""
CREATE TABLE users (
user_name varchar PRIMARY KEY,
password varchar,
gender varchar,
state varchar,
birth_year bigint
);
""")
session.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) "
"VALUES('gandalf', 'p@$$', 'male', 'WA', 1955);")
debug("Verify data is present")
session = self.patient_cql_connection(node1)
res = session.execute("SELECT * FROM Test. users")
self.assertItemsEqual(rows_to_list(res),
[[u'gandalf', 1955, u'male', u'p@$$', u'WA']])
debug("Stop node abruptly")
node1.stop(gently=False)
debug("Verify commitlog was written before abrupt stop")
commitlog_dir = os.path.join(node1.get_path(), 'commitlogs')
commitlog_files = os.listdir(commitlog_dir)
self.assertTrue(len(commitlog_files) > 0)
debug("Verify no SSTables were flushed before abrupt stop")
self.assertEqual(0, len(node1.get_sstables('test', 'users')))
debug("Verify commit log was replayed on startup")
node1.start()
node1.watch_log_for("Log replay complete")
# Here we verify from the logs that some mutations were replayed
replays = [match_tuple[0] for match_tuple in node1.grep_log(" \d+ replayed mutations")]
debug('The following log lines indicate that mutations were replayed: {msgs}'.format(msgs=replays))
num_replayed_mutations = [
parse('{} {num_mutations:d} replayed mutations{}', line).named['num_mutations']
for line in replays
]
# assert there were some lines where more than zero mutations were replayed
self.assertNotEqual([m for m in num_replayed_mutations if m > 0], [])
debug("Make query and ensure data is present")
session = self.patient_cql_connection(node1)
res = session.execute("SELECT * FROM Test. users")
self.assertItemsEqual(rows_to_list(res),
[[u'gandalf', 1955, u'male', u'p@$$', u'WA']])
def test_commitlog_replay_on_startup(self):
""" Test commit log replay """
node1 = self.node1
node1.set_configuration_options(batch_commitlog=True)
node1.start()
debug("Insert data")
session = self.patient_cql_connection(node1)
self.create_ks(session, 'Test', 1)
session.execute("""
CREATE TABLE users (
user_name varchar PRIMARY KEY,
password varchar,
gender varchar,
state varchar,
birth_year bigint
);
""")
session.execute("INSERT INTO Test. users (user_name, password, gender, state, birth_year) "
"VALUES('gandalf', 'p@$$', 'male', 'WA', 1955);")
debug("Verify data is present")
session = self.patient_cql_connection(node1)
res = session.execute("SELECT * FROM Test. users")
self.assertItemsEqual(rows_to_list(res),
[[u'gandalf', 1955, u'male', u'p@$$', u'WA']])
debug("Stop node abruptly")
node1.stop(gently=False)
debug("Verify commitlog was written before abrupt stop")
commitlog_dir = os.path.join(node1.get_path(), 'commitlogs')
commitlog_files = os.listdir(commitlog_dir)
self.assertTrue(len(commitlog_files) > 0)
debug("Verify no SSTables were flushed before abrupt stop")
for x in xrange(0, self.cluster.data_dir_count):
data_dir = os.path.join(node1.get_path(), 'data{0}'.format(x))
cf_id = [s for s in os.listdir(os.path.join(data_dir, "test")) if s.startswith("users")][0]
cf_data_dir = glob.glob("{data_dir}/test/{cf_id}".format(**locals()))[0]
cf_data_dir_files = os.listdir(cf_data_dir)
if "backups" in cf_data_dir_files:
cf_data_dir_files.remove("backups")
self.assertEqual(0, len(cf_data_dir_files))
debug("Verify commit log was replayed on startup")
node1.start()
node1.watch_log_for("Log replay complete")
# Here we verify there was more than 0 replayed mutations
zero_replays = node1.grep_log(" 0 replayed mutations")
self.assertEqual(0, len(zero_replays))
debug("Make query and ensure data is present")
session = self.patient_cql_connection(node1)
res = session.execute("SELECT * FROM Test. users")
self.assertItemsEqual(rows_to_list(res),
[[u'gandalf', 1955, u'male', u'p@$$', u'WA']])
def table_test(self):
"""
Smoke test that basic table operations work:
- create 2 tables, one with and one without COMPACT STORAGE
- ALTER the table without COMPACT STORAGE, adding a column
For each of those tables:
- insert 10 values
- SELECT * and assert the values are there
- TRUNCATE the table
- SELECT * and assert there are no values
- DROP the table
- SELECT * and assert the statement raises an InvalidRequest
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
ks_meta = UpdatingKeyspaceMetadataWrapper(session.cluster, ks_name='ks')
session.execute("CREATE TABLE test1 (k int PRIMARY KEY, v1 int)")
self.assertIn('test1', ks_meta.tables)
session.execute("CREATE TABLE test2 (k int, c1 int, v1 int, PRIMARY KEY (k, c1)) WITH COMPACT STORAGE")
self.assertIn('test2', ks_meta.tables)
t1_meta = UpdatingTableMetadataWrapper(session.cluster, ks_name='ks', table_name='test1')
session.execute("ALTER TABLE test1 ADD v2 int")
self.assertIn('v2', t1_meta.columns)
for i in range(0, 10):
session.execute("INSERT INTO test1 (k, v1, v2) VALUES ({i}, {i}, {i})".format(i=i))
session.execute("INSERT INTO test2 (k, c1, v1) VALUES ({i}, {i}, {i})".format(i=i))
res = sorted(session.execute("SELECT * FROM test1"))
self.assertEqual(rows_to_list(res), [[i, i, i] for i in range(0, 10)])
res = sorted(session.execute("SELECT * FROM test2"))
self.assertEqual(rows_to_list(res), [[i, i, i] for i in range(0, 10)])
session.execute("TRUNCATE test1")
session.execute("TRUNCATE test2")
res = session.execute("SELECT * FROM test1")
self.assertEqual(rows_to_list(res), [])
res = session.execute("SELECT * FROM test2")
self.assertEqual(rows_to_list(res), [])
session.execute("DROP TABLE test1")
self.assertNotIn('test1', ks_meta.tables)
session.execute("DROP TABLE test2")
self.assertNotIn('test2', ks_meta.tables)
def query_user(self, session, userid, age, consistency, check_ret=True):
statement = SimpleStatement(
"SELECT userid, age FROM users where userid = %d" % (userid, ),
consistency_level=consistency)
res = session.execute(statement)
expected = [[userid, age]] if age else []
ret = rows_to_list(res) == expected
if check_ret:
assert ret, "Got %s from %s, expected %s at %s" % (
rows_to_list(res), session.cluster.contact_points, expected,
self._name(consistency))
return ret
def alter_rf_and_run_read_repair_test(self):
"""
@jira_ticket CASSANDRA-10655
Data responses may skip values for columns not selected by the column filter. This can lead to empty values being
erroneously included in repair mutations sent out by the coordinator.
"""
session = self.patient_cql_connection(self.cluster.nodelist()[0])
session.execute("""CREATE KEYSPACE alter_rf_test
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};""")
session.execute("CREATE TABLE alter_rf_test.t1 (k int PRIMARY KEY, a int, b int);")
session.execute("INSERT INTO alter_rf_test.t1 (k, a, b) VALUES (1, 1, 1);")
cl_one_stmt = SimpleStatement("SELECT * FROM alter_rf_test.t1 WHERE k=1",
consistency_level=ConsistencyLevel.ONE)
# identify the initial replica and trigger a flush to ensure reads come from sstables
initial_replica, non_replicas = self.identify_initial_placement('alter_rf_test', 't1', 1)
debug("At RF=1 replica for data is " + initial_replica.name)
initial_replica.flush()
# At RF=1, it shouldn't matter which node we query, as the actual data should always come from the
# initial replica when reading at CL ONE
for n in self.cluster.nodelist():
debug("Checking " + n.name)
session = self.patient_exclusive_cql_connection(n)
res = rows_to_list(session.execute(cl_one_stmt))
assert res == [[1, 1, 1]], res
# Alter so RF=n but don't repair, then execute a query which selects only a subset of the columns. Run this at
# CL ALL on one of the nodes which doesn't currently have the data, triggering a read repair. Although we're
# only selecting a single column, the expectation is that the entire row is read on each replica to construct
# the digest responses as well as the full data reads for repair. So we expect that after the read repair, all
# replicas will have the entire row
debug("Changing RF from 1 to 3")
session.execute("""ALTER KEYSPACE alter_rf_test
WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 3};""")
cl_all_stmt = SimpleStatement("SELECT a FROM alter_rf_test.t1 WHERE k=1",
consistency_level=ConsistencyLevel.ALL)
debug("Executing SELECT on non-initial replica to trigger read repair " + non_replicas[0].name)
read_repair_session = self.patient_exclusive_cql_connection(non_replicas[0])
res = read_repair_session.execute(cl_all_stmt)
# result of the CL ALL query contains only the selected column
assert rows_to_list(res) == [[1]], res
# Now check the results of the read repair by querying each replica again at CL ONE
debug("Re-running SELECTs at CL ONE to verify read repair")
for n in self.cluster.nodelist():
debug("Checking " + n.name)
session = self.patient_exclusive_cql_connection(n)
res = rows_to_list(session.execute(cl_one_stmt))
assert res == [[1, 1, 1]], res
def table_test(self):
"""
Smoke test that basic table operations work:
- create 2 tables, one with and one without COMPACT STORAGE
- ALTER the table without COMPACT STORAGE, adding a column
For each of those tables:
- insert 10 values
- SELECT * and assert the values are there
- TRUNCATE the table
- SELECT * and assert there are no values
- DROP the table
- SELECT * and assert the statement raises an InvalidRequest
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
session.execute("CREATE TABLE test1 (k int PRIMARY KEY, v1 int)")
session.execute("CREATE TABLE test2 (k int, c1 int, v1 int, PRIMARY KEY (k, c1)) WITH COMPACT STORAGE")
session.execute("ALTER TABLE test1 ADD v2 int")
for i in range(0, 10):
session.execute("INSERT INTO test1 (k, v1, v2) VALUES ({i}, {i}, {i})".format(i=i))
session.execute("INSERT INTO test2 (k, c1, v1) VALUES ({i}, {i}, {i})".format(i=i))
res = sorted(session.execute("SELECT * FROM test1"))
self.assertEqual(rows_to_list(res), [[i, i, i] for i in range(0, 10)])
res = sorted(session.execute("SELECT * FROM test2"))
self.assertEqual(rows_to_list(res), [[i, i, i] for i in range(0, 10)])
session.execute("TRUNCATE test1")
session.execute("TRUNCATE test2")
res = session.execute("SELECT * FROM test1")
self.assertEqual(rows_to_list(res), [])
res = session.execute("SELECT * FROM test2")
self.assertEqual(rows_to_list(res), [])
session.execute("DROP TABLE test1")
session.execute("DROP TABLE test2")
assert_invalid(session, "SELECT * FROM test1", expected=InvalidRequest)
assert_invalid(session, "SELECT * FROM test2", expected=InvalidRequest)
def compact_counter_cluster_test(self):
"""
@jira_ticket CASSANDRA-12219
This test will fail on 3.0.0 - 3.0.8, and 3.1 - 3.8
"""
cluster = self.cluster
cluster.populate(3).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
self.create_ks(session, 'counter_tests', 1)
session.execute("""
CREATE TABLE IF NOT EXISTS counter_cs (
key bigint PRIMARY KEY,
data counter
) WITH COMPACT STORAGE
""")
for outer in range(0, 5):
for idx in range(0, 5):
session.execute("UPDATE counter_cs SET data = data + 1 WHERE key = {k}".format(k=idx))
for idx in range(0, 5):
row = list(session.execute("SELECT data from counter_cs where key = {k}".format(k=idx)))
self.assertEqual(rows_to_list(row)[0][0], 5)
def drop_column_compaction_test(self):
cursor = self.prepare()
cursor.execute("USE ks")
cursor.execute("CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
# insert some data.
cursor.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
cursor.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
cursor.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
cursor.execute("ALTER TABLE cf DROP c1")
cursor.execute("ALTER TABLE cf ADD c1 int")
# add another row.
cursor.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
node = self.cluster.nodelist()[0]
node.flush()
node.compact()
# erase info on dropped 'c1' column and restart.
cursor.execute("""UPDATE system.schema_columnfamilies
SET dropped_columns = null
WHERE keyspace_name = 'ks' AND columnfamily_name = 'cf'""")
node.stop(gently=False)
node.start()
time.sleep(.5)
# test that c1 values have been compacted away.
cursor = self.patient_cql_connection(node, version='3.0.10')
rows = cursor.execute("SELECT c1 FROM ks.cf")
self.assertEqual([[None], [None], [None], [4]], sorted(rows_to_list(rows)))
def compact_counter_cluster_test(self):
""" Test for bug of #12219 """
cluster = self.cluster
cluster.populate(3).start()
node1 = cluster.nodelist()[0]
session = self.patient_cql_connection(node1)
self.create_ks(session, 'counter_tests', 1)
session.execute("""
CREATE TABLE IF NOT EXISTS counter_cs (
key bigint PRIMARY KEY,
data counter
) WITH COMPACT STORAGE
""")
for outer in range(0, 5):
for idx in range(0, 5):
session.execute(
"UPDATE counter_cs SET data = data + 1 WHERE key = {k}".
format(k=idx))
for idx in range(0, 5):
row = list(
session.execute(
"SELECT data from counter_cs where key = {k}".format(
k=idx)))
self.assertEqual(rows_to_list(row)[0][0], 5)
def drop_column_compaction_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute(
"CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
node = self.cluster.nodelist()[0]
node.flush()
node.compact()
# test that c1 values have been compacted away.
session = self.patient_cql_connection(node)
rows = session.execute("SELECT c1 FROM ks.cf")
self.assertEqual([[None], [None], [None], [4]],
sorted(rows_to_list(rows)))
def cql3_insert_thrift_test(self):
""" Check that we can insert from thrift into a CQL3 table (#4377) """
session = self.prepare(start_rpc=True)
session.execute("""
CREATE TABLE test (
k int,
c int,
v int,
PRIMARY KEY (k, c)
)
""")
node = self.cluster.nodelist()[0]
host, port = node.network_interfaces['thrift']
client = get_thrift_client(host, port)
client.transport.open()
client.set_keyspace('ks')
key = struct.pack('>i', 2)
column_name_component = struct.pack('>i', 4)
# component length + component + EOC + component length + component + EOC
column_name = '\x00\x04' + column_name_component + '\x00' + '\x00\x01' + 'v' + '\x00'
value = struct.pack('>i', 8)
client.batch_mutate(
{key: {'test': [Mutation(ColumnOrSuperColumn(column=Column(name=column_name, value=value, timestamp=100)))]}},
ThriftConsistencyLevel.ONE)
res = session.execute("SELECT * FROM test")
assert rows_to_list(res) == [[2, 4, 8]], res
def assert_one(session, query, expected, cl=ConsistencyLevel.ONE):
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = rows_to_list(res)
assert list_res == [
expected
], "Expected %s from %s, but got %s" % ([expected], query, list_res)
def replace_with_reset_resume_state_test(self):
"""Test replace with resetting bootstrap progress"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
node1.start()
# restart node4 bootstrap with resetting bootstrap state
node4.stop()
mark = node4.mark_log()
node4.start(jvm_args=[
"-Dcassandra.replace_address_first_boot=127.0.0.3",
"-Dcassandra.reset_bootstrap_progress=true"
])
# check if we reset bootstrap state
node4.watch_log_for("Resetting bootstrap progress to start fresh", from_mark=mark)
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...", from_mark=mark)
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def cql3_insert_thrift_test(self):
""" Check that we can insert from thrift into a CQL3 table (#4377) """
session = self.prepare(start_rpc=True)
session.execute("""
CREATE TABLE test (
k int,
c int,
v int,
PRIMARY KEY (k, c)
)
""")
node = self.cluster.nodelist()[0]
host, port = node.network_interfaces['thrift']
client = get_thrift_client(host, port)
client.transport.open()
client.set_keyspace('ks')
key = struct.pack('>i', 2)
column_name_component = struct.pack('>i', 4)
# component length + component + EOC + component length + component + EOC
column_name = '\x00\x04' + column_name_component + '\x00' + '\x00\x01' + 'v' + '\x00'
value = struct.pack('>i', 8)
client.batch_mutate(
{key: {'test': [Mutation(ColumnOrSuperColumn(column=Column(name=column_name, value=value, timestamp=100)))]}},
ThriftConsistencyLevel.ONE)
res = session.execute("SELECT * FROM test")
assert rows_to_list(res) == [[2, 4, 8]], res
def drop_column_compaction_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute("CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
node = self.cluster.nodelist()[0]
node.flush()
node.compact()
# test that c1 values have been compacted away.
session = self.patient_cql_connection(node)
rows = session.execute("SELECT c1 FROM ks.cf")
self.assertEqual([[None], [None], [None], [4]], sorted(rows_to_list(rows)))
def many_columns_test(self):
"""
Test for tables with thousands of columns.
For CASSANDRA-11621.
"""
session = self.prepare()
width = 5000
cluster = self.cluster
session.execute("CREATE TABLE very_wide_table (pk int PRIMARY KEY, " +
",".join(map(lambda i: "c_{} int".format(i), range(width))) +
")")
session.execute("INSERT INTO very_wide_table (pk, " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
") VALUES (100," +
",".join(map(lambda i: str(i), range(width))) +
")")
res = session.execute("SELECT " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
" FROM very_wide_table")
self.assertEqual(rows_to_list(res), [[i for i in range(width)]])
def index_test(self):
"""
Smoke test CQL statements related to indexes:
- CREATE a table
- CREATE an index on that table
- INSERT 10 values into the table
- SELECT from the table over the indexed value and assert the expected values come back
- drop the index
- assert SELECTing over the indexed value raises an InvalidRequest
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
session.execute("CREATE TABLE test3 (k int PRIMARY KEY, v1 int, v2 int)")
table_meta = UpdatingTableMetadataWrapper(session.cluster, ks_name='ks', table_name='test3')
session.execute("CREATE INDEX testidx ON test3 (v1)")
self.assertIn('testidx', table_meta.indexes)
for i in range(0, 10):
session.execute("INSERT INTO test3 (k, v1, v2) VALUES ({i}, {i}, {i})".format(i=i))
res = session.execute("SELECT * FROM test3 WHERE v1 = 0")
self.assertEqual(rows_to_list(res), [[0, 0, 0]])
session.execute("DROP INDEX testidx")
self.assertNotIn('testidx', table_meta.indexes)
def many_columns_test(self):
"""
Test for tables with thousands of columns.
For CASSANDRA-11621.
"""
session = self.prepare()
width = 5000
cluster = self.cluster
session.execute(
"CREATE TABLE very_wide_table (pk int PRIMARY KEY, " +
",".join(map(lambda i: "c_{} int".format(i), range(width))) + ")")
session.execute(
"INSERT INTO very_wide_table (pk, " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
") VALUES (100," + ",".join(map(lambda i: str(i), range(width))) +
")")
res = session.execute(
"SELECT " +
",".join(map(lambda i: "c_{}".format(i), range(width))) +
" FROM very_wide_table")
self.assertEqual(rows_to_list(res), [[i for i in range(width)]])
def index_test(self):
"""
Smoke test CQL statements related to indexes:
- CREATE a table
- CREATE an index on that table
- INSERT 10 values into the table
- SELECT from the table over the indexed value and assert the expected values come back
- drop the index
- assert SELECTing over the indexed value raises an InvalidRequest
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
session.execute(
"CREATE TABLE test3 (k int PRIMARY KEY, v1 int, v2 int)")
table_meta = UpdatingTableMetadataWrapper(session.cluster,
ks_name='ks',
table_name='test3')
session.execute("CREATE INDEX testidx ON test3 (v1)")
self.assertIn('testidx', table_meta.indexes)
for i in range(0, 10):
session.execute(
"INSERT INTO test3 (k, v1, v2) VALUES ({i}, {i}, {i})".format(
i=i))
res = session.execute("SELECT * FROM test3 WHERE v1 = 0")
self.assertEqual(rows_to_list(res), [[0, 0, 0]])
session.execute("DROP INDEX testidx")
self.assertNotIn('testidx', table_meta.indexes)
def test_reading_use_header(self):
"""
Test that COPY can read a CSV with a header by:
- creating a table,
- writing a CSV with a header,
- importing the contents of the CSV file using COPY WITH HEADER = true,
- checking that the contents of the table are the written values.
"""
self.prepare()
self.session.execute("""
CREATE TABLE testheader (
a int primary key,
b int
)""")
self.tempfile = NamedTemporaryFile(delete=False)
data = [[1, 20], [2, 40], [3, 60], [4, 80]]
with open(self.tempfile.name, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=['a', 'b'])
writer.writeheader()
for a, b in data:
writer.writerow({'a': a, 'b': b})
csvfile.close
cmds = "COPY ks.testheader FROM '{name}'".format(name=self.tempfile.name)
cmds += " WITH HEADER = true"
self.node1.run_cqlsh(cmds=cmds)
result = self.session.execute("SELECT * FROM testheader")
self.assertItemsEqual([tuple(d) for d in data],
[tuple(r) for r in rows_to_list(result)])
def test_reading_use_header(self):
"""
Test that COPY can read a CSV with a header by:
- creating a table,
- writing a CSV with a header,
- importing the contents of the CSV file using COPY WITH HEADER = true,
- checking that the contents of the table are the written values.
"""
self.prepare()
self.session.execute("""
CREATE TABLE testheader (
a int primary key,
b int
)""")
self.tempfile = NamedTemporaryFile(delete=False)
data = [[1, 20], [2, 40], [3, 60], [4, 80]]
with open(self.tempfile.name, 'w') as csvfile:
writer = csv.DictWriter(csvfile, fieldnames=['a', 'b'])
writer.writeheader()
for a, b in data:
writer.writerow({'a': a, 'b': b})
csvfile.close
cmds = "COPY ks.testheader FROM '{name}'".format(
name=self.tempfile.name)
cmds += " WITH HEADER = true"
self.node1.run_cqlsh(cmds=cmds)
result = self.session.execute("SELECT * FROM testheader")
self.assertItemsEqual([tuple(d) for d in data],
[tuple(r) for r in rows_to_list(result)])
def distribution_template(self, ratio_spec, expected_ratio, delta):
"""
@param ratio_spec the string passed to `row-population-ratio` in the call to `cassandra-stress`
@param expected_ratio the expected ratio of null/non-null values in the values written
@param delta the acceptable delta between the expected and actual ratios
A parameterized test for the `row-population-ratio` parameter to
`cassandra-stress`.
"""
self.cluster.populate(1).start(wait_for_binary_proto=True)
node = self.cluster.nodelist()[0]
node.stress([
'write', 'n=1000', 'no-warmup', '-rate', 'threads=50', '-col',
'n=FIXED(50)', '-insert',
'row-population-ratio={ratio_spec}'.format(ratio_spec=ratio_spec)
])
session = self.patient_cql_connection(node)
written = rows_to_list(
session.execute('SELECT * FROM keyspace1.standard1;'))
num_nones = sum(row.count(None) for row in written)
num_results = sum(len(row) for row in written)
self.assertAlmostEqual(float(num_nones) / num_results,
expected_ratio,
delta=delta)
def resumable_replace_test(self):
"""
Test resumable bootstrap while replacing node. Feature introduced in
2.2 with ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
@jira_ticket https://issues.apache.org/jira/browse/CASSANDRA-8838
"""
cluster = self.cluster
cluster.populate(3).start()
node1, node2, node3 = cluster.nodelist()
node1.stress(['write', 'n=100K', 'no-warmup', '-schema', 'replication(factor=3)'])
session = self.patient_cql_connection(node1)
stress_table = 'keyspace1.standard1'
query = SimpleStatement('select * from %s LIMIT 1' % stress_table, consistency_level=ConsistencyLevel.THREE)
initial_data = rows_to_list(session.execute(query))
node3.stop(gently=False)
# kill node1 in the middle of streaming to let it fail
t = InterruptBootstrap(node1)
t.start()
# replace node 3 with node 4
debug("Starting node 4 to replace node 3")
node4 = Node('node4', cluster=cluster, auto_bootstrap=True, thrift_interface=('127.0.0.4', 9160),
storage_interface=('127.0.0.4', 7000), jmx_port='7400', remote_debug_port='0',
initial_token=None, binary_interface=('127.0.0.4', 9042))
# keep timeout low so that test won't hang
node4.set_configuration_options(values={'streaming_socket_timeout_in_ms': 1000})
cluster.add(node4, False)
try:
node4.start(jvm_args=["-Dcassandra.replace_address_first_boot=127.0.0.3"], wait_other_notice=False)
except NodeError:
pass # node doesn't start as expected
t.join()
# bring back node1 and invoke nodetool bootstrap to resume bootstrapping
node1.start()
node4.nodetool('bootstrap resume')
# check if we skipped already retrieved ranges
node4.watch_log_for("already available. Skipping streaming.")
# wait for node3 ready to query
node4.watch_log_for("Listening for thrift clients...")
# check if 2nd bootstrap succeeded
assert_bootstrap_state(self, node4, 'COMPLETED')
# query should work again
debug("Stopping old nodes")
node1.stop(gently=False, wait_other_notice=True)
node2.stop(gently=False, wait_other_notice=True)
debug("Verifying data on new node.")
session = self.patient_exclusive_cql_connection(node4)
assert_all(session, 'SELECT * from {} LIMIT 1'.format(stress_table),
expected=initial_data,
cl=ConsistencyLevel.ONE)
def query_counter(self, session, id, val, consistency, check_ret=True):
statement = SimpleStatement("SELECT * from counters WHERE id = %d" % (id,), consistency_level=consistency)
res = session.execute(statement)
expected = [[id, val]] if val else []
ret = rows_to_list(res) == expected
if check_ret:
assert ret, "Got %s from %s, expected %s at %s" % (res, session.cluster.contact_points, expected, self._name(consistency))
return ret
def read_counter(self, session, id, consistency):
"""
Return the current counter value. If we find no value we return zero
because after the next update the counter will become one.
"""
statement = SimpleStatement("SELECT c from counters WHERE id = {}".format(id), consistency_level=consistency)
ret = rows_to_list(session.execute(statement))
return ret[0][0] if ret else 0
def read_counter(self, session, id, consistency):
"""
Return the current counter value. If we find no value we return zero
because after the next update the counter will become one.
"""
statement = SimpleStatement("SELECT c from counters WHERE id = %d" % (id,), consistency_level=consistency)
res = rows_to_list(session.execute(statement))
return res[0][0] if res else 0
def assert_all(session, query, expected, cl=ConsistencyLevel.ONE, ignore_order=False):
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = rows_to_list(res)
if ignore_order:
expected = sorted(expected)
list_res = sorted(list_res)
assert list_res == expected, "Expected %s from %s, but got %s" % (expected, query, list_res)
def query_counter(self, session, id, val, consistency, check_ret=True):
statement = SimpleStatement("SELECT * from counters WHERE id = {}".format(id), consistency_level=consistency)
res = session.execute(statement)
ret = rows_to_list(res)
if check_ret:
self.assertEqual(ret[0][1], val, "Got {} from {}, expected {} at {}".format(ret[0][1],
session.cluster.contact_points,
val,
consistency_value_to_name(consistency)))
return ret[0][1] if ret else 0
def query_counter(self, session, id, val, consistency, check_ret=True):
statement = SimpleStatement("SELECT * from counters WHERE id = {}".format(id), consistency_level=consistency)
res = session.execute(statement)
ret = rows_to_list(res)
if check_ret:
self.assertEqual(ret[0][1], val, "Got {} from {}, expected {} at {}".format(ret[0][1],
session.cluster.contact_points,
val,
consistency_value_to_name(consistency)))
return ret[0][1] if ret else 0
def statements_test(self):
"""
INSERT, UPDATE, SELECT, SELECT COUNT, DELETE statements
"""
session = self.prepare()
session.execute(
"CREATE TABLE test7 (kind text, time int, v1 int, v2 int, PRIMARY KEY(kind, time) )"
)
for i in xrange(0, 10):
session.execute(
"INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev1', %d, %d, %d)"
% (i, i, i))
session.execute(
"INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev2', %d, %d, %d)"
% (i, i, i))
res = session.execute("SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'")
assert rows_to_list(res) == [[10]], res
res = session.execute(
"SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2')")
assert rows_to_list(res) == [[20]], res
res = session.execute(
"SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2') AND time=0"
)
assert rows_to_list(res) == [[2]], res
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev1'")
assert rows_to_list(res) == [['ev1', i, i, i]
for i in xrange(0, 10)], res
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev2'")
assert rows_to_list(res) == [['ev2', i, i, i]
for i in xrange(0, 10)], res
for i in xrange(0, 10):
session.execute(
"UPDATE test7 SET v1 = 0, v2 = 0 where kind = 'ev1' AND time=%d"
% (i, ))
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev1'")
assert rows_to_list(res) == [['ev1', i, 0, 0]
for i in xrange(0, 10)], res
res = session.execute("DELETE FROM test7 WHERE kind = 'ev1'")
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev1'")
assert rows_to_list(res) == [], res
res = session.execute("SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'")
assert rows_to_list(res) == [[0]], res
def _test_bulk_round_trip(self, nodes, partitioner, num_records):
"""
Test exporting a large number of rows into a csv file.
"""
self.prepare(nodes=nodes, partitioner=partitioner)
self.node1.stress(
['write', 'n={}'.format(num_records), '-rate', 'threads=50'])
stress_table = 'keyspace1.standard1'
self.assertEqual(
[[num_records]],
rows_to_list(
self.session.execute(
"SELECT COUNT(*) FROM {}".format(stress_table))))
self.tempfile = NamedTemporaryFile(delete=False)
debug('Exporting to csv file: {}'.format(self.tempfile.name))
start = datetime.datetime.now()
self.node1.run_cqlsh(
cmds="COPY {} TO '{}'".format(stress_table, self.tempfile.name))
debug("COPY TO took {} to export {} records".format(
datetime.datetime.now() - start, num_records))
# check all records were exported
self.assertEqual(num_records,
sum(1 for line in open(self.tempfile.name)))
self.session.execute("TRUNCATE {}".format(stress_table))
debug('Importing from csv file: {}'.format(self.tempfile.name))
start = datetime.datetime.now()
self.node1.run_cqlsh(
cmds="COPY {} FROM '{}'".format(stress_table, self.tempfile.name))
debug("COPY FROM took {} to import {} records".format(
datetime.datetime.now() - start, num_records))
self.assertEqual(
[[num_records]],
rows_to_list(
self.session.execute(
"SELECT COUNT(*) FROM {}".format(stress_table))))
def query_counter(self, session, id, val, consistency, check_ret=True):
statement = SimpleStatement("SELECT * from counters WHERE id = %d" %
(id, ),
consistency_level=consistency)
res = session.execute(statement)
ret = rows_to_list(res)
if check_ret:
assert ret[0][1] == val, "Got %s from %s, expected %s at %s" % (
ret[0][1], session.cluster.contact_points, val,
self._name(consistency))
return ret[0][1] if ret else 0
def _insert_rows(session, table_name, insert_stmt, values):
prepared_insert = session.prepare(insert_stmt)
values = list(values) # in case values is a generator
execute_concurrent(session, ((prepared_insert, x) for x in values),
concurrency=500, raise_on_first_error=True)
data_loaded = rows_to_list(session.execute('SELECT * FROM ' + table_name))
debug('{n} rows inserted into {table_name}'.format(n=len(data_loaded), table_name=table_name))
# use assert_equal over assert_length_equal to avoid printing out
# potentially large lists
assert_equal(len(values), len(data_loaded))
return data_loaded
def test_query_indexes_with_vnodes(self):
"""
Verifies correct query behaviour in the presence of vnodes
@jira_ticket CASSANDRA-11104
"""
cluster = self.cluster
cluster.populate(2).start()
node1, node2 = cluster.nodelist()
session = self.patient_cql_connection(node1)
session.execute("CREATE KEYSPACE ks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': '1'};")
session.execute("CREATE TABLE ks.compact_table (a int PRIMARY KEY, b int) WITH COMPACT STORAGE;")
session.execute("CREATE INDEX keys_index ON ks.compact_table (b);")
session.execute("CREATE TABLE ks.regular_table (a int PRIMARY KEY, b int)")
session.execute("CREATE INDEX composites_index on ks.regular_table (b)")
def index_is_built(table_name, idx_name):
index_query = (
"""SELECT * FROM system_schema.indexes WHERE keyspace_name = 'ks' AND table_name = '{}' AND index_name = '{}'""".format(table_name, idx_name)
if self.cluster.version() > '3.0' else
"""SELECT * FROM system."IndexInfo" WHERE table_name = 'ks' AND index_name = '{}.{}'""".format(table_name, idx_name)
)
return len(list(session.execute(index_query))) == 1
start = time.time()
while not index_is_built('regular_table', 'composites_index') and time.time() + 10 < start:
debug("waiting for index to build")
time.sleep(1)
insert_args = [(i, i % 2) for i in xrange(100)]
execute_concurrent_with_args(session,
session.prepare("INSERT INTO ks.compact_table (a, b) VALUES (?, ?)"),
insert_args)
execute_concurrent_with_args(session,
session.prepare("INSERT INTO ks.regular_table (a, b) VALUES (?, ?)"),
insert_args)
res = session.execute("SELECT * FROM ks.compact_table WHERE b = 0")
self.assertEqual(len(rows_to_list(res)), 50)
res = session.execute("SELECT * FROM ks.regular_table WHERE b = 0")
self.assertEqual(len(rows_to_list(res)), 50)
def statements_test(self):
"""
Smoke test SELECT and UPDATE statements:
- create a table
- insert 20 rows into the table
- run SELECT COUNT queries and assert they return the correct values
- bare and with IN and equality conditions
- run SELECT * queries with = conditions
- run UPDATE queries
- SELECT * and assert the UPDATEd values are there
- DELETE with a = condition
- SELECT the deleted values and make sure nothing is returned
# TODO run SELECTs to make sure each statement works
"""
session = self.prepare()
session.execute("CREATE TABLE test7 (kind text, time int, v1 int, v2 int, PRIMARY KEY(kind, time) )")
for i in range(0, 10):
session.execute("INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev1', {i}, {i}, {i})".format(i=i))
session.execute("INSERT INTO test7 (kind, time, v1, v2) VALUES ('ev2', {i}, {i}, {i})".format(i=i))
res = session.execute("SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'")
self.assertEqual(rows_to_list(res), [[10]])
res = session.execute("SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2')")
self.assertEqual(rows_to_list(res), [[20]])
res = session.execute("SELECT COUNT(*) FROM test7 WHERE kind IN ('ev1', 'ev2') AND time=0")
self.assertEqual(rows_to_list(res), [[2]])
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev1'")
self.assertEqual(rows_to_list(res), [['ev1', i, i, i] for i in range(0, 10)])
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev2'")
self.assertEqual(rows_to_list(res), [['ev2', i, i, i] for i in range(0, 10)])
for i in range(0, 10):
session.execute("UPDATE test7 SET v1 = 0, v2 = 0 where kind = 'ev1' AND time={i}".format(i=i))
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev1'")
self.assertEqual(rows_to_list(res), [['ev1', i, 0, 0] for i in range(0, 10)])
session.execute("DELETE FROM test7 WHERE kind = 'ev1'")
res = session.execute("SELECT * FROM test7 WHERE kind = 'ev1'")
self.assertEqual(rows_to_list(res), [])
res = session.execute("SELECT COUNT(*) FROM test7 WHERE kind = 'ev1'")
self.assertEqual(rows_to_list(res), [[0]])
def gc_test(self):
"""
Test that tombstone purging doesn't bring back deleted data by writing
2 rows to a table with gc_grace=0, deleting one of those rows, then
asserting that it isn't present in the results of SELECT *, before and
after a flush and compaction.
"""
cluster = self.cluster
cluster.populate(1).start()
[node1] = cluster.nodelist()
time.sleep(.5)
session = self.patient_cql_connection(node1)
self.create_ks(session, 'ks', 1)
self.create_cf(session,
'cf',
gc_grace=0,
key_type='int',
columns={'c1': 'int'})
session.execute('insert into cf (key, c1) values (1,1)')
session.execute('insert into cf (key, c1) values (2,1)')
node1.flush()
self.assertEqual(rows_to_list(session.execute('select * from cf;')),
[[1, 1], [2, 1]])
session.execute('delete from cf where key=1')
self.assertEqual(rows_to_list(session.execute('select * from cf;')),
[[2, 1]])
node1.flush()
time.sleep(.5)
node1.compact()
time.sleep(.5)
self.assertEqual(rows_to_list(session.execute('select * from cf;')),
[[2, 1]])
def _token_gen_test(self, nodes, randomPart=None):
generated_tokens, session = self.prepare(randomPart, nodes=nodes)
dc_tokens = generated_tokens[0]
tokens = []
local_tokens = rows_to_list(session.execute("SELECT tokens FROM system.local"))[0]
self.assertEqual(local_tokens.__len__(), 1, "too many tokens for peer")
for tok in local_tokens:
tokens += tok
rows = rows_to_list(session.execute("SELECT tokens FROM system.peers"))
self.assertEqual(rows.__len__(), nodes - 1)
for row in rows:
peer_tokens = row[0]
self.assertEqual(peer_tokens.__len__(), 1, "too many tokens for peer")
for tok in peer_tokens:
tokens.append(tok)
self.assertEqual(tokens.__len__(), dc_tokens.__len__())
for cluster_token in tokens:
tok = int(cluster_token)
self.assertGreaterEqual(dc_tokens.index(tok), 0, "token in cluster does not match generated tokens")
def assert_all(session,
query,
expected,
cl=ConsistencyLevel.ONE,
ignore_order=False):
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = rows_to_list(res)
if ignore_order:
expected = sorted(expected)
list_res = sorted(list_res)
assert list_res == expected, "Expected %s from %s, but got %s" % (
expected, query, list_res)
def _insert_rows(session, table_name, insert_stmt, values):
prepared_insert = session.prepare(insert_stmt)
values = list(values) # in case values is a generator
execute_concurrent(session, ((prepared_insert, x) for x in values),
concurrency=500,
raise_on_first_error=True)
data_loaded = rows_to_list(session.execute('SELECT * FROM ' + table_name))
debug('{n} rows inserted into {table_name}'.format(n=len(data_loaded),
table_name=table_name))
# use assert_equal over assert_length_equal to avoid printing out
# potentially large lists
assert_equal(len(values), len(data_loaded))
return data_loaded
def drop_column_queries_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute("CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
session.execute("CREATE INDEX ON cf(c2)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
# test that old (pre-drop) c1 values aren't returned and new ones are.
rows = session.execute("SELECT c1 FROM cf")
self.assertEqual([[None], [None], [None], [4]], sorted(rows_to_list(rows)))
rows = session.execute("SELECT * FROM cf")
self.assertEqual([[0, None, 2], [1, None, 3], [2, None, 4], [3, 4, 5]], sorted(rows_to_list(rows)))
rows = session.execute("SELECT c1 FROM cf WHERE key = 0")
self.assertEqual([[None]], rows_to_list(rows))
rows = session.execute("SELECT c1 FROM cf WHERE key = 3")
self.assertEqual([[4]], rows_to_list(rows))
rows = session.execute("SELECT * FROM cf WHERE c2 = 2")
self.assertEqual([[0, None, 2]], rows_to_list(rows))
rows = session.execute("SELECT * FROM cf WHERE c2 = 5")
self.assertEqual([[3, 4, 5]], rows_to_list(rows))
def assert_none(session, query, cl=None):
"""
Assert query returns nothing
@param session Session to use
@param query Query to run
@param cl Optional Consistency Level setting. Default ONE
Examples:
assert_none(self.session1, "SELECT * FROM test where key=2;")
assert_none(cursor, "SELECT * FROM test WHERE k=2", cl=ConsistencyLevel.SERIAL)
"""
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = tools.rows_to_list(res)
assert list_res == [], "Expected nothing from {}, but got {}".format(query, list_res)
def assert_none(session, query, cl=None):
"""
Assert query returns nothing
@param session Session to use
@param query Query to run
@param cl Optional Consistency Level setting. Default ONE
Examples:
assert_none(self.session1, "SELECT * FROM test where key=2;")
assert_none(cursor, "SELECT * FROM test WHERE k=2", cl=ConsistencyLevel.SERIAL)
"""
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = tools.rows_to_list(res)
assert list_res == [], "Expected nothing from {}, but got {}".format(
query, list_res)
def index_test(self):
"""
CREATE INDEX, DROP INDEX statements
"""
session = self.prepare()
session.execute("CREATE TABLE test3 (k int PRIMARY KEY, v1 int, v2 int)")
session.execute("CREATE INDEX testidx ON test3 (v1)")
for i in xrange(0, 10):
session.execute("INSERT INTO test3 (k, v1, v2) VALUES (%d, %d, %d)" % (i, i, i))
res = session.execute("SELECT * FROM test3 WHERE v1 = 0")
assert rows_to_list(res) == [[0, 0, 0]], res
session.execute("DROP INDEX testidx")
assert_invalid(session, "SELECT * FROM test3 where v1 = 0", expected=InvalidRequest)
def drop_counter_column_test(self):
"""Test for CASSANDRA-7831"""
cluster = self.cluster
cluster.populate(1).start()
node1, = cluster.nodelist()
session = self.patient_cql_connection(node1)
self.create_ks(session, 'counter_tests', 1)
session.execute("CREATE TABLE counter_bug (t int, c counter, primary key(t))")
session.execute("UPDATE counter_bug SET c = c + 1 where t = 1")
row = list(session.execute("SELECT * from counter_bug"))
self.assertEqual(rows_to_list(row)[0], [1, 1])
self.assertEqual(len(row), 1)
session.execute("ALTER TABLE counter_bug drop c")
assert_invalid(session, "ALTER TABLE counter_bug add c counter", "Cannot re-add previously dropped counter column c")
def assert_one(session, query, expected, cl=None):
"""
Assert query returns one row.
@param session Session to use
@param query Query to run
@param expected Expected results from query
@param cl Optional Consistency Level setting. Default ONE
Examples:
assert_one(session, "LIST USERS", ['cassandra', True])
assert_one(session, query, [0, 0])
"""
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = tools.rows_to_list(res)
assert list_res == [expected
], "Expected {} from {}, but got {}".format([expected],
query,
list_res)
def assert_all(session, query, expected, cl=None, ignore_order=False):
"""
Assert query returns all expected items optionally in the correct order
@param session Session in use
@param query Query to run
@param expected Expected results from query
@param cl Optional Consistency Level setting. Default ONE
@param ignore_order Optional boolean flag determining whether response is ordered
Examples:
assert_all(session, "LIST USERS", [['aleksey', False], ['cassandra', True]])
assert_all(self.session1, "SELECT * FROM ttl_table;", [[1, 42, 1, 1]])
"""
simple_query = SimpleStatement(query, consistency_level=cl)
res = session.execute(simple_query)
list_res = tools.rows_to_list(res)
if ignore_order:
expected = sorted(expected)
list_res = sorted(list_res)
assert list_res == expected, "Expected {} from {}, but got {}".format(
expected, query, list_res)
def drop_column_queries_test(self):
session = self.prepare()
session.execute("USE ks")
session.execute(
"CREATE TABLE cf (key int PRIMARY KEY, c1 int, c2 int)")
session.execute("CREATE INDEX ON cf(c2)")
# insert some data.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (0, 1, 2)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (1, 2, 3)")
session.execute("INSERT INTO cf (key, c1, c2) VALUES (2, 3, 4)")
# drop and readd c1.
session.execute("ALTER TABLE cf DROP c1")
session.execute("ALTER TABLE cf ADD c1 int")
# add another row.
session.execute("INSERT INTO cf (key, c1, c2) VALUES (3, 4, 5)")
# test that old (pre-drop) c1 values aren't returned and new ones are.
rows = session.execute("SELECT c1 FROM cf")
self.assertEqual([[None], [None], [None], [4]],
sorted(rows_to_list(rows)))
rows = session.execute("SELECT * FROM cf")
self.assertEqual([[0, None, 2], [1, None, 3], [2, None, 4], [3, 4, 5]],
sorted(rows_to_list(rows)))
rows = session.execute("SELECT c1 FROM cf WHERE key = 0")
self.assertEqual([[None]], rows_to_list(rows))
rows = session.execute("SELECT c1 FROM cf WHERE key = 3")
self.assertEqual([[4]], rows_to_list(rows))
rows = session.execute("SELECT * FROM cf WHERE c2 = 2")
self.assertEqual([[0, None, 2]], rows_to_list(rows))
rows = session.execute("SELECT * FROM cf WHERE c2 = 5")
self.assertEqual([[3, 4, 5]], rows_to_list(rows))
def test_commitlog_replay_on_startup(self):
"""
Test commit log replay
"""
node1 = self.node1
node1.set_batch_commitlog(enabled=True)
node1.start()
debug("Insert data")
session = self.patient_cql_connection(node1)
self.create_ks(session, 'Test', 1)
session.execute("""
CREATE TABLE users (
user_name varchar PRIMARY KEY,
password varchar,
gender varchar,
state varchar,
birth_year bigint
);
""")
session.execute(
"INSERT INTO Test. users (user_name, password, gender, state, birth_year) "
"VALUES('gandalf', 'p@$$', 'male', 'WA', 1955);")
debug("Verify data is present")
session = self.patient_cql_connection(node1)
res = session.execute("SELECT * FROM Test. users")
self.assertItemsEqual(rows_to_list(res),
[[u'gandalf', 1955, u'male', u'p@$$', u'WA']])
debug("Stop node abruptly")
node1.stop(gently=False)
debug("Verify commitlog was written before abrupt stop")
commitlog_dir = os.path.join(node1.get_path(), 'commitlogs')
commitlog_files = os.listdir(commitlog_dir)
self.assertTrue(len(commitlog_files) > 0)
debug("Verify no SSTables were flushed before abrupt stop")
self.assertEqual(0, len(node1.get_sstables('test', 'users')))
debug("Verify commit log was replayed on startup")
node1.start()
node1.watch_log_for("Log replay complete")
# Here we verify from the logs that some mutations were replayed
replays = [
match_tuple[0]
for match_tuple in node1.grep_log(" \d+ replayed mutations")
]
debug(
'The following log lines indicate that mutations were replayed: {msgs}'
.format(msgs=replays))
num_replayed_mutations = [
parse('{} {num_mutations:d} replayed mutations{}',
line).named['num_mutations'] for line in replays
]
# assert there were some lines where more than zero mutations were replayed
self.assertNotEqual([m for m in num_replayed_mutations if m > 0], [])
debug("Make query and ensure data is present")
session = self.patient_cql_connection(node1)
res = session.execute("SELECT * FROM Test. users")
self.assertItemsEqual(rows_to_list(res),
[[u'gandalf', 1955, u'male', u'p@$$', u'WA']])
